From 7be90742ed71b4327789ad36753f0e69195dd295 Mon Sep 17 00:00:00 2001 From: Jorge Samuel Mendes de Jesus Date: Mon, 19 Apr 2021 16:37:42 +0200 Subject: [PATCH] Dockerfile/K8s implementation, prod/dev run (#24) --- .dockerignore | 16 +++++++++++++++ .env.default | 4 ++-- .gitignore | 5 +++++ Dockerfile | 29 ++++++++++++++++++++++++++ README.md | 37 +++++++++++++++++++++++++++------- components/footer/component.js | 2 +- components/map/constants.js | 2 +- index.js | 1 + run.sh | 32 +++++++++++++++++++++++++++++ 9 files changed, 117 insertions(+), 11 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 run.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..329282c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +# ignore folder on copy (more for local build) +*.md +.git +.cache +.history +node_modules +npm-debug.log +.prettierrc +.gitignore +.github +.git +.eslintrc +.env.default +.editorconfig +.dockerignore +.env diff --git a/.env.default b/.env.default index 7c1e2ce..eafb378 100644 --- a/.env.default +++ b/.env.default @@ -1,11 +1,11 @@ -PORT=3000 +PORT=3001 MAPBOX_API_KEY= API_URL= ANALYSIS_API_URL= -DEPLOYMENT_KEY= GOOGLE_ANALYTICS_KEY= AWS_REGION= AWS_BUCKET_NAME= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_MAX_Z_TILE_STORAGE= +DEPLOYMENT_KEY= diff --git a/.gitignore b/.gitignore index 756b8f6..d89545e 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,8 @@ yarn-error.log* # keys gee.key.json + +#VScode +.history/ + + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ede5796 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM docker.io/debian:stable-slim@sha256:a939c03c4d3e3f53e3ef4ef6e75cb681a3ad56537842f95bf89755da86559b13 + +# Source: https://www.kabisa.nl/tech/nvm-in-docker/ +# docker build --force-rm --no-cache -t soils-revealed:latest . +# docker run -p3001:3001 --env-file .env soils-revealed:latest + +SHELL ["/bin/bash","-l","-c"] +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update \ + && apt-get install -y tini curl \ + && rm -rf /var/lib/apt/lists/* + +RUN mkdir /soils-revealed +WORKDIR /soils-revealed +COPY . . + +RUN curl --silent -o- https://raw.githubusercontent.com/creationix/nvm/master/install.sh | bash +RUN nvm install && nvm use +RUN echo -e "Implementing node: $(node --version)\nFrom: $(which node)" + +RUN npm install -g yarn +RUN npm install -g pm2 +RUN yarn install --frozen-lockfile + +RUN DEPLOYMENT_KEY=$(date +%s) && echo "DEPLOYMENT_KEY=$DEPLOYMENT_KEY" > .env + +ENTRYPOINT ["/usr/bin/tini","-g","--"] +CMD ["/soils-revealed/run.sh","production"] diff --git a/README.md b/README.md index a21af21..97ab754 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,15 @@ In order to start modifying the app, please make sure to correctly configure you 5. Use the correct Node.js version for this app by running `nvm use`; if you didn't install NVM (step 2), then manually install the Node.js version described in `.nvmrc` 6. Install the dependencies: `yarn` 7. Create a `.env` file at the root of the project by copying `.env.default` and giving a value for each of the variables (see next section for details) -8. Create a `gee.key.json` file at the root of the project with the Google Earth Engine's private key inside +8. Create a gee.key.json file at the root of the project with the Google Earth Engine's private key inside. + 9. Run the server: `yarn dev` You can access a hot-reloaded version of the app on [http://localhost:3000](http://localhost:3000). The application is built using [React](https://reactjs.org/) and the framework [Next.js](https://nextjs.org/). The styles use [Sass](https://sass-lang.com/) and the [Bootstrap](https://getbootstrap.com/) framework. -A continuous deployment system is in place. Each time you push to the `master` branch, the application is deployed to production through a GitHub Action defined in `.github/workflows/production.yml`. Each time you push to `develop`, the application is deployed to staging through an action defined in `.github/workflows/staging.yml`. You can see the status of the build in the “Actions” tab of the repository on GitHub. +A continuous deployment system is in place. Each time you push to the `master` branch, the application is deployed to production. Github will trigger an event on google cloud run and make a deployment. The same for the `develop` branch. ## Environment variables @@ -47,15 +48,37 @@ Below is a description of each of the keys. ## Deployment -As explained before, the application is automatically deployed to staging when pushing new changes to the `develop` branch, and deployed to production when pushing changes to `master`. This is achieved through GitHub Actions defined in `.github/workflows`. +### Local computer + +It is possible to run a local test deployment using the docker image. Docker implement an agnostic build and then during run it will pickup the container's env variables, and properly set the system + +`run.sh` accepts 2 arguments: production or develop. Production argument will run `yarn start` and the code will production ready, while `develop` runs nodejs in development mode, necessary if you are testing content and changes. + +```bash +docker build --force-rm --no-cache -t soils-revealed:latest . +docker run -p3001:3001 --env-file .env soils-revealed:latest /soils-revealed/run.sh production +``` + +**Note:** We have created a `.env` file on the project root with all variables, and this will be used to run nodejs in production mode. This is a deployment to test nodejs in production mode. + +**Note:** Dockerfile has `CMD` implementing production. + + +### Google GKE -When an action is executed, it connects via SSH to the server hosting the application. The server's credentials are stored in GitHub's “secrets” vault. A script is then executed: the running instance of the application is stopped, the code is pulled, the correct version of node is selected, the dependencies are installed, a local `.env` file is generated, a local `gee.key.json` file is generated, and the application is restarted. +Public deployment is based on Google Cloud build and Google GKE (Kubernetes). Up on push to `master` or `develop`, the following steps will happen: -The `.env` file is programmatically generated on the server because it differs for each environment. Some of the keys are hard coded in the `.github/workflows/XXX.yml` file and others are pulled from GitHub's “secrets” vault. +1. Github will trigger a Google Cloud run trigger +2. Google cloud will pull the branch content. +3. Docker build will be initicated. +4. After completed Docker image is stored on a private repository. +5. Image will then be deplyed into the soils-revealed cluster. +6. GKE contains a specific `ConfigMap` with all .env necessary for deployment. +7. `gee.key.json` is added to the pods using a `ConfigMap` mount -The `gee.key.json` file contains the credentials for the Google Earth Engine library. It is also programmatically generated as its value is stored in GitHub's “secrets” vault. +GKE will implement the available Dockerfile. -Overall, deploying to either environment takes between 1 to 2 minutes to complete. +Overall, deploying to either environment takes between 5 to 10 minutes to complete. ## Architecture diff --git a/components/footer/component.js b/components/footer/component.js index 0151bbd..4708091 100644 --- a/components/footer/component.js +++ b/components/footer/component.js @@ -42,7 +42,7 @@ const Footer = () => {
  • - + Contact
    • diff --git a/components/map/constants.js b/components/map/constants.js index 5dbdeb9..2457ebe 100644 --- a/components/map/constants.js +++ b/components/map/constants.js @@ -1082,7 +1082,7 @@ exports.LAYERS = { 'soc-experimental': { label: 'Soil organic carbon experimental approach', description: - 'These maps display soil organic carbon in Argentina in any year between 1982 and 2017. You can toggle between soil organic carbon concentration and soil organic carbon stock, as well as between soil organic carbon maps for any given year and change maps between years. Selecting “Change” will display where Argentina has experienced a gain in soil organic carbon and which parts a loss for a chosen time period. To produce these maps, we used a novel machine learning algorithm based on field samples to map soil organic carbon over space and time.\n\nA global implementation of this approach is in process. If you would like to join this effort, please [contact us](mailto:soilsrevealed@tnc.org), and join our growing partnership.', + 'These maps display soil organic carbon in Argentina in any year between 1982 and 2017. You can toggle between soil organic carbon concentration and soil organic carbon stock, as well as between soil organic carbon maps for any given year and change maps between years. Selecting “Change” will display where Argentina has experienced a gain in soil organic carbon and which parts a loss for a chosen time period. To produce these maps, we used a novel machine learning algorithm based on field samples to map soil organic carbon over space and time.\n\nA global implementation of this approach is in process. If you would like to join this effort, please [contact us](mailto:info@soilsrevealed.org), and join our growing partnership.', group: 'soc', attributions: [], bbox: [ diff --git a/index.js b/index.js index 66b4c51..a2f920b 100644 --- a/index.js +++ b/index.js @@ -36,6 +36,7 @@ const handle = app.getRequestHandler(); // Initialize GEE let geePrivateKey; try { + geePrivateKey = require('./gee.key.json'); ee.data.authenticateViaPrivateKey( geePrivateKey, diff --git a/run.sh b/run.sh new file mode 100755 index 0000000..f7af374 --- /dev/null +++ b/run.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +yarn_build () { + echo -e "Using: $(cat .env)" + source /root/.nvm/nvm.sh + echo -e "Preparing .env file" + echo -e "PORT=$PORT\nMAPBOX_API_KEY=$MAPBOX_API_KEY\nAPI_URL=$API_URL\nANALYSIS_API_URL=$ANALYSIS_API_URL\nGOOGLE_ANALYTICS_KEY=$GOOGLE_ANALYTICS_KEY\nAWS_REGION=$AWS_REGION\nAWS_BUCKET_NAME=$AWS_BUCKET_NAME\nAWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID\nAWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY\nAWS_MAX_Z_TILE_STORAGE=$AWS_MAX_Z_TILE_STORAGE" >> .env + echo -e "Initiating yarn build" + + yarn build + +} + + +case "$1" in + production) + yarn_build + exec pm2 start --no-daemon yarn -- start + ;; + develop) + yarn_build + exec node index.js + ;; + *) + echo >&2 "Invalid option: $@ \n either production or develop"; exit 1 + ;; +esac + +#node index.js +#yarn start +#exec pm2 start --no-daemon yarn -- start +