-
Notifications
You must be signed in to change notification settings - Fork 4
/
Copy path.gitlab-ci.yml
186 lines (181 loc) · 10.7 KB
/
.gitlab-ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
image: python:3.8.0b4
services:
- docker:dind
stages:
- Test and Deploy
before_script:
# Setup package manager
- apt-get update 2>&1 >/dev/null
- apt-get install curl wget git -y
- apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran -y
- apt-get install python -y
- apt-get install python-pip -y
- apt-get install python-setuptools -y
- apt-get install python3-setuptools python3-dev python3-venv -y
- apt-get install python-dev -y
- apt-get install openssh-server -y
# Setup SSH configuration
- mkdir -p ~/.ssh
- echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config
# Install gettext for envsubst
- apt-get install gettext -y
# Install AWSCLI
- pip install awscli
Test and Deploy:
stage: Test and Deploy
script:
# Install Coldbrew dependencies
- apt-get update
# Install Java
- apt-get install default-jre cmake ninja-build -y
# Install Emscripten
- ./src/tools/install_emscripten.sh
# Install Python dependencies
- apt-get install build-essential -y
- apt-get install libffi-dev libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev -y
- apt-get install libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev liblzma-dev tk-dev -y
# Install OpenSSL with thread support
- wget http://www.openssl.org/source/openssl-1.0.1g.tar.gz
- tar -xvzf openssl-1.0.1g.tar.gz
- cd openssl-1.0.1g && ./config threads -D_REENTRANT
- cd openssl-1.0.1g && make
- cd openssl-1.0.1g && make install_sw
- rm -rf openssl-1.0.1g.tar.gz openssl-1.0.1g
# Install Java for Closure Compiler
- (cd /opt && curl -L https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u222-b10/OpenJDK8U-jre_x64_linux_hotspot_8u222b10.tar.gz -o jre.tar.gz && tar zxvf jre.tar.gz --one-top-level=jre)
- rm /opt/jre.tar.gz
- echo "" >> /usr/local/coldbrew/emsdk/emsdk_env.sh
- echo "PATH=/opt/jre/jdk8u222-b10-jre/bin:\$PATH" >> /usr/local/coldbrew/emsdk/emsdk_env.sh
# Install Makefile dependencies
- apt-get install zip unzip rsync -y
# Install wasm-nm and wasm-gc
- curl https://sh.rustup.rs -sSf | sh -s -- -y
- /bin/bash -c "source $HOME/.cargo/env; cargo install wasm-nm"
- /bin/bash -c "source $HOME/.cargo/env; cargo install wasm-gc"
# Install uglify-js and Google Closure Compiler
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; npm install -g uglify-es google-closure-compiler"
# Increase node memory size
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; mv \$EMSDK_NODE \$EMSDK_NODE.bak"
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; echo -e '#!/usr/bin/env bash' >> \$EMSDK_NODE"
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; echo -e \"\$EMSDK_NODE.bak --max_old_space_size=16384 \\\"\\\$@\\\"\" >> \$EMSDK_NODE"
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; chmod +x \$EMSDK_NODE"
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cat \$EMSDK_NODE"
# Patch some emscripten files
- cd $CI_PROJECT_DIR
# Install Chrome pre-requisites
- wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add -
- sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
- apt-get update
- apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst --no-install-recommends
# Add GitLab SSH private deploy key
- eval $(ssh-agent -s)
- tmpfile=$(mktemp ~/pk.XXXXXX)
- echo "$SSH_PRIVATE_KEY" > $tmpfile
- ssh-add $tmpfile
- rm $tmpfile
# Tag the release on GitLab
- rm -rf ../tagger
- mkdir -p ../tagger
- cd ../tagger
- git clone [email protected]:$CI_PROJECT_PATH.git
- cd $CI_PROJECT_NAME
- git config --global user.email "[email protected]"
- git config --global user.name "Plasticity Developer Bot"
# Build
- export COLDBREW_TAGGER_DIR=$(pwd)
- python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); print(flags&os.O_NONBLOCK);'
- python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); fcntl.fcntl(sys.stdout, fcntl.F_SETFL, flags&~os.O_NONBLOCK);'
- python -c 'import os,sys,fcntl; flags = fcntl.fcntl(sys.stdout, fcntl.F_GETFL); print(flags&os.O_NONBLOCK);'
- cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/src; make; cd ../
# Build Debug and Build Node
- cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/src; echo "" >> ../customize/coldbrew_settings.py; echo "DEBUG_MODE = True" >> ../customize/coldbrew_settings.py; COLDBREW_DIST_DIR=dist-debug ASYNC_DEBUG=1 make; cd ../
- git checkout ./customize/coldbrew_settings.py
# ##### Clean before Node build
- cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/src; make cleanlocal; cd ../
- cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/src; echo "" >> ../customize/coldbrew_settings.py; echo "NODE = True" >> ../customize/coldbrew_settings.py; COLDBREW_DIST_DIR=dist-node make; cd ../
- git checkout ./customize/coldbrew_settings.py
- cat .gitignore | grep 'dist' -v > .gitignore.tmp
- mv .gitignore.tmp .gitignore
# End Build
# Test
- cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/; npm install; echo 'Running tests...'; ./test.sh
# Push to NPM registry
- echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; npm whoami"
- /bin/bash -c "cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /$COLDBREW_TAGGER_DIR/; cd dist-node; npm publish --access=public; cd .."
- rm -rf dist-node/**/* || true
- rm -rf dist-node/* || true
- echo "You can download this version from https://registry.npmjs.org/@plasticity/coldbrew/-/coldbrew-$(python -c "import version; print(version.__version__)").tgz" >> dist-node/download_instructions.txt
# End Push to NPM registry
# Commit
- git pull
- git add -A
- git status
- git commit -n -m "Release $(python -c "import version; print(version.__version__)")"
- git push -f origin HEAD:releases
# End Commit
# Push Tags
- git checkout releases
- git pull
- git tag --delete latest || true
- git tag -a latest -m "Release Latest"
- git show latest --pretty=format:"%H" --no-patch
- git tag -a $(python -c "import version; print(version.__version__)") -m "Release $(python -c "import version; print(version.__version__)")"
- git show $(python -c "import version; print(version.__version__)") --pretty=format:"%H" --no-patch
- git push -f origin HEAD:releases --tags
# End Push Tags
# Push to Amazon S3
- sed -e "s/localhost:8000/coldbrew.plasticity.ai/g; s/coldbrew.js/https:\/\/cdn.jsdelivr.net\/gh\/plasticityai\/coldbrew@$(python -c "import version; print(version.__version__)")\/dist\/coldbrew.js/g;" dist/index.html > dist/index.remote.html
- sed -e "s/localhost:8000/coldbrew.plasticity.ai/g; s/coldbrew.js/https:\/\/cdn.jsdelivr.net\/gh\/plasticityai\/coldbrew@$(python -c "import version; print(version.__version__)")\/dist-debug\/coldbrew.js/g;" dist/index.html > dist/index.remote-debug.html
- aws s3 cp dist/index.remote.html s3://coldbrew.plasticity.ai/index.html
- aws s3 cp dist/index.remote-debug.html s3://coldbrew.plasticity.ai/index-debug.html
- aws s3 cp dist/index.remote.html s3://coldbrew.plasticity.ai/index-$(python -c "import version; print(version.__version__)").html
- aws s3 cp dist/index.remote-debug.html s3://coldbrew.plasticity.ai/index-$(python -c "import version; print(version.__version__)")-debug.html
- aws s3 cp src/remote s3://coldbrew.plasticity.ai/remote/ --recursive
- aws s3 cp images s3://coldbrew.plasticity.ai/images/ --recursive
# End Push to Amazon S3
- cd $CI_PROJECT_DIR
- rm -rf ../tagger
- sleep 60 # Wait for GitLab to mirror to GitHub
# Create a release on GitHub
- curl -u plasticity-admin:$GITHUB_TOKEN -d "{\"tag_name\":\"$(python -c "import version; print(version.__version__)")\", \"name\":\"Release $(python -c "import version; print(version.__version__)")\"}" -H "Content-Type:"" application/json" -X POST https://api.github.com/repos/plasticityai/$CI_PROJECT_NAME/releases
environment:
name: Coldbrew Demo
url: http://coldbrew.plasticity.ai/
only:
- master
Build Docker Image:
stage: Test and Deploy
image: registry.gitlab.com/plasticity/ubuntu-dind:16.04
script:
- compressed=$(mktemp ~/compressed.XXXXXX)
- cd $CI_PROJECT_DIR
- docker login -u "gitlab-ci-token" -p "$CI_BUILD_TOKEN" $CI_REGISTRY
- docker pull "$CI_REGISTRY_IMAGE:latest" || true
- docker build --pull --cache-from "$CI_REGISTRY_IMAGE:latest" -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA" .
- docker run "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA" sleep 1
- docker export $(docker ps -aq | head -n1) > $compressed
- docker import $compressed "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA"
- docker history "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA"
- rm $compressed
- docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA"
- docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA" "$CI_REGISTRY_IMAGE:$(python3 -c "import version; print(version.__version__)")"
- docker push "$CI_REGISTRY_IMAGE:$(python3 -c "import version; print(version.__version__)")"
- docker tag "$CI_REGISTRY_IMAGE:$CI_COMMIT_SHA" "$CI_REGISTRY_IMAGE:latest"
- docker push "$CI_REGISTRY_IMAGE:latest"
- echo -e "\nCOPY . /BUILD/\n\nRUN /bin/bash -c \"cd /usr/local/coldbrew/emsdk; source ./emsdk_env.sh; cd /BUILD/src; make python;\"" >> Dockerfile
- rm .dockerignore
- docker build --pull --cache-from "$CI_REGISTRY_IMAGE:latest" -t "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA" .
- docker run "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA" sleep 1
- docker export $(docker ps -aq | head -n1) > $compressed
- docker import $compressed "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA"
- docker history "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA"
- rm $compressed
- docker push "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA"
- docker push "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA"
- docker tag "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA" "$CI_REGISTRY_IMAGE/builder:$(python3 -c "import version; print(version.__version__)")"
- docker push "$CI_REGISTRY_IMAGE/builder:$(python3 -c "import version; print(version.__version__)")"
- docker tag "$CI_REGISTRY_IMAGE/builder:$CI_COMMIT_SHA" "$CI_REGISTRY_IMAGE/builder:latest"
- docker push "$CI_REGISTRY_IMAGE/builder:latest"
only:
- master