From 53e794b088d2f1a8feef357fba61e211e497ff02 Mon Sep 17 00:00:00 2001 From: rolljee Date: Tue, 28 Nov 2023 10:41:27 +0100 Subject: [PATCH] Update workflows, merge repository kuzzle-containers --- .github/workflows/core-dev.workflow.yml | 38 +++++++ .github/workflows/elasticsearch.workflow.yml | 40 ++++++++ .github/workflows/kuzzle-runner.workflow.yml | 101 ++++++++++++++++++ .github/workflows/workflow-deployments.yaml | 95 +++++++++++------ .github/workflows/workflow.yaml | 102 +++++++++++++------ README.md | 2 +- docker/docker-compose-swarm.yml | 100 ------------------ docker/images/core-dev/Dockerfile | 32 ++++++ docker/images/core-dev/README.md | 21 ++++ docker/images/elasticsearch/Dockerfile | 13 +++ docker/images/elasticsearch/README.md | 7 ++ docker/images/elasticsearch/jvm.options | 94 +++++++++++++++++ docker/images/kuzzle-runner/Dockerfile | 27 +++++ package.json | 48 ++++----- 14 files changed, 526 insertions(+), 194 deletions(-) create mode 100644 .github/workflows/core-dev.workflow.yml create mode 100644 .github/workflows/elasticsearch.workflow.yml create mode 100644 .github/workflows/kuzzle-runner.workflow.yml delete mode 100644 docker/docker-compose-swarm.yml create mode 100644 docker/images/core-dev/Dockerfile create mode 100644 docker/images/core-dev/README.md create mode 100644 docker/images/elasticsearch/Dockerfile create mode 100644 docker/images/elasticsearch/README.md create mode 100644 docker/images/elasticsearch/jvm.options create mode 100644 docker/images/kuzzle-runner/Dockerfile diff --git a/.github/workflows/core-dev.workflow.yml b/.github/workflows/core-dev.workflow.yml new file mode 100644 index 0000000000..e11ee75b98 --- /dev/null +++ b/.github/workflows/core-dev.workflow.yml @@ -0,0 +1,38 @@ +name: kuzzleio/core-dev + +on: + push: + branches: + - master + +env: + DOCKER_PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7" + +jobs: + build-and-push: + name: Build and push core-dev image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./core-dev + file: ./core-dev/Dockerfile + push: true + platforms: ${{ env.DOCKER_PLATFORMS }} + tags: kuzzleio/core-dev:2,kuzzleio/core-dev:latest diff --git a/.github/workflows/elasticsearch.workflow.yml b/.github/workflows/elasticsearch.workflow.yml new file mode 100644 index 0000000000..8ed9278392 --- /dev/null +++ b/.github/workflows/elasticsearch.workflow.yml @@ -0,0 +1,40 @@ +name: kuzzleio/elasticsearch + +on: + push: + branches: + - master + +env: + DOCKER_PLATFORMS: "linux/amd64,linux/arm64" + ELASTICSEARCH_VERSION: "7.16.2" + +jobs: + build-and-push: + name: Build and push elasticsearch image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v2 + with: + context: ./elasticsearch + file: ./elasticsearch/Dockerfile + push: true + build-args: ELASTICSEARCH_VERSION=${{ env.ELASTICSEARCH_VERSION }} + platforms: ${{ env.DOCKER_PLATFORMS }} + tags: kuzzleio/elasticsearch:${{ env.ELASTICSEARCH_VERSION }},kuzzleio/elasticsearch:7, kuzzleio/elasticsearch:latest diff --git a/.github/workflows/kuzzle-runner.workflow.yml b/.github/workflows/kuzzle-runner.workflow.yml new file mode 100644 index 0000000000..712a355d0a --- /dev/null +++ b/.github/workflows/kuzzle-runner.workflow.yml @@ -0,0 +1,101 @@ +name: kuzzleio/kuzzle-runner + +on: + push: + branches: + - master + +env: + DOCKER_PLATFORMS: "linux/amd64,linux/arm64,linux/arm/v7" + NODE_LTS_MAINTENANCE_VERSION: "16" + NODE_LTS_ACTIVE_VERSION: "18" + NODE_LTS_CURRENT_VERSION: "20" + +jobs: + maintenance: + name: Build and deploy image embedding Node.js maintenance LTS + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./kuzzle-runner + file: ./kuzzle-runner/Dockerfile + push: true + build-args: NODE_LTS_VERSION=${{ env.NODE_LTS_MAINTENANCE_VERSION }} + platforms: ${{ env.DOCKER_PLATFORMS }} + tags: kuzzleio/kuzzle-runner:${{ env.NODE_LTS_MAINTENANCE_VERSION }},kuzzleio/kuzzle-runner:maintenance + + active: + name: Build and deploy image embedding Node.js active LTS version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: ./kuzzle-runner + file: ./kuzzle-runner/Dockerfile + push: true + build-args: NODE_LTS_VERSION=${{ env.NODE_LTS_ACTIVE_VERSION }} + platforms: ${{ env.DOCKER_PLATFORMS }} + tags: kuzzleio/kuzzle-runner:${{ env.NODE_LTS_ACTIVE_VERSION }},kuzzleio/kuzzle-runner:active,kuzzleio/kuzzle-runner:latest + + # Node 20 is not currently supported, this will be uncommented in a upcomming PR + # current: + # name: Build and deploy image embedding Node.js latest LTS version + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v3 + + # - name: Set up QEMU + # uses: docker/setup-qemu-action@v3 + + # - name: Set up Docker Buildx + # id: buildx + # uses: docker/setup-buildx-action@v3 + + # - name: Login to DockerHub + # uses: docker/login-action@v3 + # with: + # username: ${{ secrets.DOCKER_USERNAME }} + # password: ${{ secrets.DOCKER_PASSWORD }} + + # - name: Build and push + # uses: docker/build-push-action@v3 + # with: + # context: ./kuzzle-runner + # file: ./kuzzle-runner/Dockerfile + # push: true + # build-args: NODE_LTS_VERSION=${{ env.NODE_LTS_CURRENT_VERSION }} + # platforms: ${{ env.DOCKER_PLATFORMS }} + # tags: kuzzleio/kuzzle-runner:${{ env.NODE_LTS_CURRENT_VERSION }},kuzzleio/kuzzle-runner:current diff --git a/.github/workflows/workflow-deployments.yaml b/.github/workflows/workflow-deployments.yaml index 036b82f08d..d79c5fdc1b 100644 --- a/.github/workflows/workflow-deployments.yaml +++ b/.github/workflows/workflow-deployments.yaml @@ -4,20 +4,23 @@ on: workflow_call: inputs: node_lts_maintenance_version: - description: "Current Node LTS Version" + description: "Maintenance Node LTS Version" required: true default: "16" type: string - node_lts_current_version: - description: "Current Node LTS Version" - required: true - default: "20" - type: string + node_lts_active_version: description: "Active Node LTS Version" required: true default: "18" type: string + + node_lts_current_version: + description: "Current Node LTS Version" + required: true + default: "20" + type: string + docker_platforms: description: "Docker platforms" required: true @@ -30,31 +33,37 @@ on: required: true default: true type: boolean + dockerhub_deploy: description: "Deploy to DockerHub" required: true default: true type: boolean + npm_deploy: description: "Deploy to NPM" required: true default: true type: boolean + node_lts_maintenance_version: - description: "Current Node LTS Version" + description: "Maintenance Node LTS Version" required: true default: "16" type: string - node_lts_current_version: - description: "Current Node LTS Version" - required: true - default: "20" - type: string + node_lts_active_version: description: "Active Node LTS Version" required: true default: "18" type: string + + node_lts_current_version: + description: "Current Node LTS Version" + required: true + default: "20" + type: string + docker_platforms: description: "Docker platforms" required: true @@ -70,16 +79,18 @@ jobs: if: ${{ (github.event_name != 'workflow_dispatch' || inputs.doc_deploy) && (github.ref_name == 'master' || github.ref_name == '2-dev') }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - name: Checkout project + uses: actions/checkout@v3 + - name: Extract references from context shell: bash id: extract-refs run: | - echo "::set-output name=version::$(git describe --abbrev=0 --tags | cut -d. -f 1)" - echo "::set-output name=repo::$(echo $GITHUB_REPOSITORY | cut -d/ -f 2)" - - uses: convictional/trigger-workflow-and-wait@v1.6.3 + echo "version=$(git describe --abbrev=0 --tags | cut -d. -f 1)" >> $GITHUB_OUTPUT + echo "repo=$(echo $GITHUB_REPOSITORY | cut -d/ -f 2)" >> $GITHUB_OUTPUT + + - name: Trigger documentation workflow + uses: convictional/trigger-workflow-and-wait@v1.6.3 with: owner: kuzzleio repo: documentation @@ -93,35 +104,43 @@ jobs: if: ${{ (github.event_name != 'workflow_dispatch' || inputs.dockerhub_deploy) && github.ref_name == 'master' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 - with: - node-version: ${{ inputs.node_lts_maintenance_version }} - - uses: actions/cache@v3 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ inputs.node_lts_active_version }} + uses: actions/setup-node@v4 with: - path: "**/node_modules" - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} + node-version: ${{ inputs.node_lts_active_version }} + cache: 'npm' + - name: Set up QEMU uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx id: buildx uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get current version from package.json shell: bash id: get-version run: | - echo "::set-output name=version::$(jq -r .version package.json)" - echo "::set-output name=major-version::$(jq -r .version package.json | cut -d. -f 1)" + echo "version=$(jq -r .version package.json)" >> $GITHUB_OUTPUT + echo "major-version=$(jq -r .version package.json | cut -d. -f 1)" >> $GITHUB_OUTPUT + - name: Build TS files run: | npm ci npm run build + - name: Build and push uses: docker/build-push-action@v2 with: @@ -136,13 +155,23 @@ jobs: if: ${{ (github.event_name != 'workflow_dispatch' || inputs.npm_deploy) && github.ref_name == 'master' }} runs-on: ubuntu-latest steps: + - name: Checkout project - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ inputs.node_lts_active_version }} + - uses: actions/setup-node@v4 with: - node-version: ${{ inputs.node_lts_maintenance_version }} + node-version: ${{ inputs.node_lts_active_version }} registry-url: "https://registry.npmjs.org" - - run: npm ci - - run: npm publish + cache: 'npm' + + - name: Install depedencies + run: npm ci + + - name: Publish package + run: npm publish env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index d2b51e070c..bdf920d1f8 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -5,7 +5,7 @@ on: branches: - master - 2-dev - pull_request: # This syntax stand for all PRs events + pull_request: env: NODE_LTS_MAINTENANCE_VERSION: "16" @@ -20,7 +20,7 @@ jobs: steps: - id: set-matrix run: | - echo "::set-output name=matrix::{\"node-version\": [\"$NODE_LTS_MAINTENANCE_VERSION\", \"$NODE_LTS_ACTIVE_VERSION\"]}" + echo "matrix={\"node-version\": [\"$NODE_LTS_MAINTENANCE_VERSION\", \"$NODE_LTS_ACTIVE_VERSION\"]}" >> $GITHUB_OUTPUT outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} node_lts_maintenance_version: ${{ env.NODE_LTS_MAINTENANCE_VERSION }} @@ -32,9 +32,15 @@ jobs: name: Documentation - Error codes check runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - run: npm ci + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Install depedencies + run: npm ci + - name: Test error codes run: ./.ci/scripts/check-error-codes-documentation.sh @@ -46,13 +52,20 @@ jobs: matrix: node-version: ${{ fromJson(needs.prepare-matrix.outputs.matrix).node-version }} steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - - uses: ./.github/actions/es-lint + cache: "npm" + + - name: Run ESlint + uses: ./.github/actions/es-lint unit-tests: name: Unit Tests @@ -62,13 +75,20 @@ jobs: node-version: ${{ fromJson(needs.prepare-matrix.outputs.matrix).node-version }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - - uses: ./.github/actions/unit-tests + cache: "npm" + + - name: Run unit test using Node ${{ matrix.node-version }} + uses: ./.github/actions/unit-tests env: NODE_VERSION: ${{ matrix.node-version }} @@ -85,13 +105,20 @@ jobs: node-version: ${{ fromJson(needs.prepare-matrix.outputs.matrix).node-version }} kuzzle-image: ["kuzzle"] steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - - uses: ./.github/actions/build-and-run-kuzzle + cache: "npm" + + - name: Test to run Kuzzle image + uses: ./.github/actions/build-and-run-kuzzle with: KUZZLE_IMAGE: ${{ matrix.kuzzle-image }} @@ -112,13 +139,20 @@ jobs: node-version: ${{ fromJson(needs.prepare-matrix.outputs.matrix).node-version }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages - - uses: actions/setup-node@v3 + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + + - name: Setup Node version ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - - uses: ./.github/actions/functional-tests + cache: "npm" + + - name: Run functional test suit ${{ matrix.test_set }} + uses: ./.github/actions/functional-tests with: test-set: test:functional:${{ matrix.test_set }} node-version: ${{ matrix.node-version }} @@ -133,18 +167,26 @@ jobs: matrix: node-version: ${{ fromJson(needs.prepare-matrix.outputs.matrix).node-version }} steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/install-packages + - name: Checkout project + uses: actions/checkout@v3 + + - name: Install additional libraries + uses: ./.github/actions/install-packages + - name: Cloning Monkey Tester uses: actions/checkout@v3 with: repository: kuzzleio/kuzzle-monkey-tests path: "kuzzle-monkey-tests" - - uses: actions/setup-node@v3 + + - name: Setup Node version ${{ matrix.node-version }} + uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - key: ${{ runner.os }}-${{ matrix.node-version }}-${{ hashFiles('**/package-lock.json') }} - - uses: ./.github/actions/monkey-tests + cache: "npm" + + - name: Run monkey testing tests suit + uses: ./.github/actions/monkey-tests with: node-version: ${{ matrix.node-version }} env: diff --git a/README.md b/README.md index c8e8f27b7c..28dc3d7c1f 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ First of all make sure that you have at least `4GB` of ram allocated to your vm Run the following command to install all the dependencies in your container: ```bash -npm run docker:install +docker compose run kuzzle_node_1 npm ci ``` Finally, run the command `docker compose up` to start your Kuzzle stack. diff --git a/docker/docker-compose-swarm.yml b/docker/docker-compose-swarm.yml deleted file mode 100644 index 353c20cae9..0000000000 --- a/docker/docker-compose-swarm.yml +++ /dev/null @@ -1,100 +0,0 @@ -version: '3.4' - -services: - traefik: - image: traefik - container_name: traefik - ports: - - "7512:7512" - - "1883:1883" - command: - - --log.level=WARN - - --providers.docker=true - - --providers.docker.exposedbydefault=false - - --providers.docker.endpoint=unix:///var/run/docker.sock - - --entrypoints.kuzzle_http.address=:7512 - - --entrypoints.kuzzle_mqtt.address=:1883 - depends_on: - kuzzle: - condition: service_healthy - volumes: - - "/var/run/docker.sock:/var/run/docker.sock:ro" - - kuzzle: - image: kuzzleio/core-dev:2 - command: node app.js - volumes: - - "..:/var/app" - - "./scripts/run-dev.sh:/run-dev.sh" - - "./fixtures:/fixtures" - cap_add: - - SYS_PTRACE - ulimits: - nofile: 65536 - sysctls: - - net.core.somaxconn=8192 - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:7512/_healthcheck"] - timeout: 1s - interval: 2s - retries: 30 - depends_on: - redis: - condition: service_healthy - elasticsearch: - condition: service_healthy - labels: - - "traefik.enable=true" - # Kuzzle HTTP/WS - - "traefik.http.services.kuzzle_http.loadbalancer.server.port=7512" - - "traefik.http.services.kuzzle_http.loadbalancer.healthCheck.path=/_healthcheck" - - "traefik.http.services.kuzzle_http.loadbalancer.healthCheck.interval=5s" - - "traefik.http.routers.kuzzle.entrypoints=kuzzle_http" - - "traefik.http.routers.kuzzle.rule=Host(`localhost`)" - - "traefik.http.routers.kuzzle.service=kuzzle_http" - # Kuzzle MQTT - - "traefik.tcp.services.kuzzle_mqtt.loadbalancer.server.port=1883" - - "traefik.tcp.routers.kuzzle.entrypoints=kuzzle_mqtt" - - "traefik.tcp.routers.kuzzle.rule=HostSNI(`*`)" - - "traefik.tcp.routers.kuzzle.service=kuzzle_mqtt" - environment: - - kuzzle_services__storageEngine__client__node=http://elasticsearch:9200 - - kuzzle_services__storageEngine__commonMapping__dynamic=true - - kuzzle_services__internalCache__node__host=redis - - kuzzle_services__memoryStorage__node__host=redis - - kuzzle_server__protocols__mqtt__enabled=true - - kuzzle_server__protocols__mqtt__developmentMode=false - - kuzzle_limits__loginsPerSecond=50 - # - NODE_ENV=${NODE_ENV:-development} - - NODE_ENV=production - # - DEBUG=${DEBUG:-kuzzle:*,-kuzzle:network:protocols:websocket,-kuzzle:events} - - DEBUG=${DEBUG:-kuzzle:cluster:sync} - - DEBUG_DEPTH=${DEBUG_DEPTH:-0} - - DEBUG_MAX_ARRAY_LENGTH=${DEBUG_MAX_ARRAY:-100} - - DEBUG_EXPAND=${DEBUG_EXPAND:-off} - - DEBUG_SHOW_HIDDEN={$DEBUG_SHOW_HIDDEN:-on} - - DEBUG_COLORS=${DEBUG_COLORS:-on} - - redis: - image: redis:6.2.4 - container_name: kuzzle_redis - healthcheck: - test: ["CMD", "redis-cli", "ping"] - interval: 1s - timeout: 3s - retries: 30 - ports: - - "6379:6379" - - elasticsearch: - image: kuzzleio/elasticsearch:7 - container_name: kuzzle_elasticsearch - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9200"] - interval: 2s - timeout: 2s - retries: 10 - ports: - - "9200:9200" - ulimits: - nofile: 65536 diff --git a/docker/images/core-dev/Dockerfile b/docker/images/core-dev/Dockerfile new file mode 100644 index 0000000000..25cba5b446 --- /dev/null +++ b/docker/images/core-dev/Dockerfile @@ -0,0 +1,32 @@ +ARG NODE_LTS_VERSION="18" +FROM node:${NODE_LTS_VERSION}-bullseye-slim + +LABEL io.kuzzle.vendor="Kuzzle " +LABEL description="Enhance the Kuzzle core with ease" + +RUN mkdir /var/npm && chown node:node /var/npm + +ENV NPM_CONFIG_CACHE=/var/npm + +WORKDIR /var/app + +RUN set -x \ + \ + && apt-get update \ + && apt-get install -y \ + bash-completion \ + build-essential \ + curl \ + g++ \ + gdb \ + git \ + gnupg \ + libfontconfig \ + libzmq3-dev \ + python3 \ + procps \ + wget \ + libunwind-dev \ + && npm install -g kourou \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/docker/images/core-dev/README.md b/docker/images/core-dev/README.md new file mode 100644 index 0000000000..e82814f03a --- /dev/null +++ b/docker/images/core-dev/README.md @@ -0,0 +1,21 @@ +# kuzzleio/core-dev + +*~ 720 MB.* +This image is useful for core developer. Kuzzle source code is mounted inside the container. + +Contains: + * all needed to watch and build kuzzleio projects on the fly: + * NVM with node 18 + * build-essential + * g++ + * python + * rbenv + * libfontconfig + * [gem] sass + * [npm] bower + * [npm] node-inspector + * tools to improve development experience: + * bash-completion + * ll alias + * curl + * wget diff --git a/docker/images/elasticsearch/Dockerfile b/docker/images/elasticsearch/Dockerfile new file mode 100644 index 0000000000..fe8d79ff46 --- /dev/null +++ b/docker/images/elasticsearch/Dockerfile @@ -0,0 +1,13 @@ +ARG ELASTICSEARCH_VERSION="7.16.2" +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION} + +LABEL maintainer="kuzzle" contact="" + +COPY jvm.options /usr/share/elasticsearch/config/jvm.options + +ENV ES_JAVA_OPTS="-Xms512m -Xmx512m" + +ENV cluster.name=kuzzle +ENV node.name=alyx +ENV discovery.type=single-node +ENV ingest.geoip.downloader.enabled=false diff --git a/docker/images/elasticsearch/README.md b/docker/images/elasticsearch/README.md new file mode 100644 index 0000000000..95b9c18d44 --- /dev/null +++ b/docker/images/elasticsearch/README.md @@ -0,0 +1,7 @@ +# Elasticsearch + +This container is built from [Elasticsearch official 7.10 image](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html) with a few modifications: + +* All xpack plugins are disabled by default +* `jvm.options` file is modified to remove the jvm heap size settings, thus allowing to set them with `ES_JAVA_OPTS` environment variable (default to 512m). +* The GeoIP downloader is disabled by default diff --git a/docker/images/elasticsearch/jvm.options b/docker/images/elasticsearch/jvm.options new file mode 100644 index 0000000000..7a5ab342cb --- /dev/null +++ b/docker/images/elasticsearch/jvm.options @@ -0,0 +1,94 @@ +################################################################ +## +## JVM configuration +## +################################################################ +## +## WARNING: DO NOT EDIT THIS FILE. If you want to override the +## JVM options in this file, or set any additional options, you +## should create one or more files in the jvm.options.d +## directory containing your adjustments. +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/jvm-options.html +## for more information. +## +################################################################ + + + +################################################################ +## IMPORTANT: JVM heap size +################################################################ +## +## The heap size is automatically configured by Elasticsearch +## based on the available memory in your system and the roles +## each node is configured to fulfill. If specifying heap is +## required, it should be done through a file in jvm.options.d, +## and the min and max should be set to the same value. For +## example, to set the heap to 4 GB, create a new file in the +## jvm.options.d directory containing these lines: +## +## -Xms4g +## -Xmx4g +## +## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html +## for more information +## +################################################################ + + +################################################################ +## Expert settings +################################################################ +## +## All settings below here are considered expert settings. Do +## not adjust them unless you understand what you are doing. Do +## not edit them in this file; instead, create a new file in the +## jvm.options.d directory containing your adjustments. +## +################################################################ + +## GC configuration +8-13:-XX:+UseConcMarkSweepGC +8-13:-XX:CMSInitiatingOccupancyFraction=75 +8-13:-XX:+UseCMSInitiatingOccupancyOnly + +## G1GC Configuration +# NOTE: G1 GC is only supported on JDK version 10 or later +# to use G1GC, uncomment the next two lines and update the version on the +# following three lines to your version of the JDK +# 10-13:-XX:-UseConcMarkSweepGC +# 10-13:-XX:-UseCMSInitiatingOccupancyOnly +14-:-XX:+UseG1GC + +## JVM temporary directory + +## heap dumps + +# generate a heap dump when an allocation from the Java heap fails; heap dumps +# are created in the working directory of the JVM unless an alternative path is +# specified +-XX:+HeapDumpOnOutOfMemoryError + +# specify an alternative path for heap dumps; ensure the directory exists and +# has sufficient space +-XX:HeapDumpPath=data + +# specify an alternative path for JVM fatal error logs +-XX:ErrorFile=logs/hs_err_pid%p.log + +## JDK 8 GC logging +8:-XX:+PrintGCDetails +8:-XX:+PrintGCDateStamps +8:-XX:+PrintTenuringDistribution +8:-XX:+PrintGCApplicationStoppedTime +8:-Xloggc:logs/gc.log +8:-XX:+UseGCLogFileRotation +8:-XX:NumberOfGCLogFiles=32 +8:-XX:GCLogFileSize=64m + +# JDK 9+ GC logging +9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m + +# Log4Shell mitigation +-Dlog4j2.formatMsgNoLookups=true diff --git a/docker/images/kuzzle-runner/Dockerfile b/docker/images/kuzzle-runner/Dockerfile new file mode 100644 index 0000000000..ba87e94f2e --- /dev/null +++ b/docker/images/kuzzle-runner/Dockerfile @@ -0,0 +1,27 @@ +ARG NODE_LTS_VERSION="18" +FROM node:${NODE_LTS_VERSION}-bullseye-slim + +LABEL io.kuzzle.vendor="Kuzzle " +LABEL description="Run your in-development Kuzzle application with ease" + +RUN mkdir /var/npm && chown node:node /var/npm + +ENV NPM_CONFIG_CACHE=/var/npm + +WORKDIR /var/app + +RUN set -x \ + && apt-get update && apt-get install -y \ + curl \ + python3 \ + make \ + g++ \ + git \ + libzmq3-dev \ + libunwind-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN npm i -g kourou + +WORKDIR /var/app diff --git a/package.json b/package.json index e73aea61a4..882b1a19bc 100644 --- a/package.json +++ b/package.json @@ -9,32 +9,21 @@ "clean": "touch index.ts && npm run build | grep TSFILE | cut -d' ' -f 2 | xargs rm", "codecov": "codecov", "cucumber": "cucumber.js --fail-fast", - "dev:test": "npm run dev -- docker/scripts/start-kuzzle-test.ts --enable-plugins functional-test-plugin", - "dev": "npx ergol docker/scripts/start-kuzzle-dev.ts -c ./config/ergol.config.json", + "dev": "ergol docker/scripts/start-kuzzle-dev.ts -c ./config/ergol.config.json", "doc-error-codes": "node -r ts-node/register doc/build-error-codes", - "docker:install": "docker compose run kuzzle_node_1 npm ci", - "docker:npm": "docker compose run kuzzle_node_1 npm run --", - "docker:test:unit": "docker compose run kuzzle_node_1 npm run test:unit", - "docker": "docker compose run kuzzle_node_1 ", - "file": "npx ergol docker/scripts/start-kuzzle-dev.ts -c ./config/ergol.config.json", "prepublishOnly": "npm run build", - "prettier": "npx prettier ./lib ./test ./bin ./features ./plugins/available/functional-test-plugin --write", - "services": "npx kourou app:start-services", - "start:dev": "npx ergol docker/scripts/start-kuzzle-dev.ts -c ./config/ergol.config.json --script-args=--enable-plugins functional-test-plugin", - "test:functional:http": "KUZZLE_PROTOCOL=http npx cucumber-js --profile http", + "prettier": "prettier ./lib ./test ./bin ./features ./plugins/available/functional-test-plugin --write", + "test:functional:http": "KUZZLE_PROTOCOL=http cucumber-js --profile http", "test:functional:jest": "jest", - "test:functional:legacy:http": "npx cucumber-js --format progress-bar --profile http ./features-legacy", - "test:functional:legacy:mqtt": "npx cucumber-js --format progress-bar --profile mqtt ./features-legacy", - "test:functional:legacy:websocket": "npx cucumber-js --format progress-bar --profile websocket ./features-legacy", + "test:functional:legacy:http": "cucumber-js --format progress-bar --profile http ./features-legacy", + "test:functional:legacy:mqtt": "cucumber-js --format progress-bar --profile mqtt ./features-legacy", + "test:functional:legacy:websocket": "cucumber-js --format progress-bar --profile websocket ./features-legacy", "test:functional:legacy": "npm run test:functional:legacy:http && npm run test:functional:legacy:websocket && npm run test:functional:legacy:mqtt", - "test:functional:websocket": "KUZZLE_PROTOCOL=websocket npx cucumber-js --profile websocket", + "test:functional:websocket": "KUZZLE_PROTOCOL=websocket cucumber-js --profile websocket", "test:functional": "npm run test:functional:http && npm run test:functional:websocket && npm run test:functional:jest", - "test:lint:js:fix": "eslint --max-warnings=0 --fix ./lib ./test ./bin ./features", "test:lint:js": "eslint --max-warnings=0 ./lib ./test ./bin ./features ./plugins/available/functional-test-plugin", - "test:lint:ts:fix": "eslint --max-warnings=0 --fix ./lib --ext .ts --config .eslintc-ts.json", "test:lint:ts": "eslint --max-warnings=0 ./lib --ext .ts --config .eslintc-ts.json", "test:lint": "npm run test:lint:js && npm run test:lint:ts", - "test:unit:coverage": "DEBUG= nyc --reporter=text-summary --reporter=lcov mocha --exit", "test:unit": "DEBUG= npx --node-arg=--trace-warnings mocha --exit", "test": "npm run clean && npm run --silent test:lint && npm run build && npm run test:unit:coverage && npm run test:functional" }, @@ -52,8 +41,8 @@ "didyoumean": "1.2.2", "dumpme": "1.0.3", "eventemitter3": "4.0.7", - "inquirer": "9.1.4", - "ioredis": "5.3.0", + "inquirer": "9.2.12", + "ioredis": "5.3.2", "js-yaml": "4.1.0", "json-stable-stringify": "1.0.2", "json2yaml": "1.1.0", @@ -61,27 +50,27 @@ "koncorde": "4.3.0", "kuzzle-plugin-auth-passport-local": "6.4.0", "kuzzle-plugin-logger": "3.0.3", - "kuzzle-sdk": "7.11.0", + "kuzzle-sdk": "7.11.1", "kuzzle-vault": "2.0.4", "lodash": "4.17.21", - "long": "5.2.1", + "long": "5.2.3", "moment": "2.29.4", "ms": "2.1.3", "murmurhash-native": "3.5.0", "ndjson": "2.0.0", "node-segfault-handler": "1.4.2", "passport": "0.6.0", - "protobufjs": "~7.2.1", + "protobufjs": "7.2.5", "rc": "1.2.8", - "semver": "7.3.8", + "semver": "7.5.4", "sorted-array": "2.0.4", "uuid": "9.0.1", - "uWebSockets.js": "https://github.com/uNetworking/uWebSockets.js/archive/refs/tags/v20.32.0.tar.gz", + "uWebSockets.js": "https://github.com/uNetworking/uWebSockets.js/archive/refs/tags/v20.34.0.tar.gz", "validator": "13.11.0", - "winston": "3.10.0", + "winston": "3.11.0", "winston-elasticsearch": "0.17.4", "winston-syslog": "2.7.0", - "winston-transport": "4.5.0", + "winston-transport": "4.6.0", "yargs": "17.7.2", "zeromq": "6.0.0-beta.6" }, @@ -99,11 +88,10 @@ "codecov": "3.8.3", "cucumber": "6.0.5", "ergol": "1.0.2", - "eslint-plugin-kuzzle": "0.0.6", + "eslint-plugin-kuzzle": "0.0.12", "jest": "29.7.0", "mocha": "10.2.0", "mock-require": "3.0.3", - "mqtt": "4.3.7", "nyc": "15.1.0", "request": "2.88.2", "request-promise": "4.2.6", @@ -112,7 +100,7 @@ "should-sinon": "0.0.6", "sinon": "14.0.2", "strip-json-comments": "https://github.com/sindresorhus/strip-json-comments/archive/refs/tags/v3.1.1.tar.gz", - "ts-jest": "29.0.5", + "ts-jest": "29.1.1", "ts-node": "10.9.1", "typescript": "4.9.5", "yaml": "2.2.1"