diff --git a/.dockerignore b/.dockerignore index 666f0a234..887bb9542 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,3 @@ -# comment -cache/* -git_cache/* -Docker/**/*.tar.gz test/suites/**/results test/log test/mwlog diff --git a/.github/actions/prep-env-and-cache/action.yml b/.github/actions/prep-env-and-cache/action.yml deleted file mode 100644 index 5ea080c2f..000000000 --- a/.github/actions/prep-env-and-cache/action.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: 'checkout-and-prepare' -description: 'Prepare our github actions environment' - -runs: - using: "composite" - steps: - - name: Print the env from variables.env - run: cat variables.env - shell: bash - - - name: Set the env from variables.env - uses: c-py/action-dotenv-to-setenv@v4 - with: - env-file: variables.env - - # https://phabricator.wikimedia.org/T346882 - - name: Cache composer files - id: cache-composer - uses: actions/cache@v3 - env: - cache-name: cache-wikibase-composer - with: - path: cache - key: cache-wikibase-composer - - - shell: bash - run: | - echo "Composer Cache Hit: ${{ steps.cache-composer.outputs.cache-hit }}" - - - name: Cache files in git_cache - id: cache-git - uses: actions/cache@v3 - env: - cache-name: cache-wikibase-git-repo - with: - path: git_cache - key: cache-wikibase-git-repo - - - shell: bash - run: | - echo "Git Cache Hit: ${{ steps.cache-git.outputs.cache-hit }}" - diff --git a/.github/actions/upload-results/action.yml b/.github/actions/upload-results/action.yml index fd9548c7f..362be2a82 100644 --- a/.github/actions/upload-results/action.yml +++ b/.github/actions/upload-results/action.yml @@ -9,23 +9,18 @@ inputs: runs: using: "composite" steps: - - name: Archive build_metadata Artifacts (if any) - uses: actions/upload-artifact@v3 - with: - name: Metadata - path: artifacts/build_metadata_*.env - if-no-files-found: ignore - - name: Archive tar artifacts (if any) uses: actions/upload-artifact@v3 with: name: TarBalls - path: artifacts/${{ inputs.component }}.tar.gz + path: | + artifacts/${{ inputs.component }}*.tar.gz + !artifacts/${{ inputs.component }}*.docker.tar.gz if-no-files-found: ignore - name: Archive Docker artifact uses: actions/upload-artifact@v3 with: name: DockerImages - path: artifacts/${{ inputs.component }}.docker.tar.gz + path: artifacts/${{ inputs.component }}*.docker.tar.gz if-no-files-found: error diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 371beeae7..d91062003 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -46,11 +46,8 @@ jobs: - name: Checkout the repo uses: actions/checkout@v4 - - name: Prepare environment and caches - uses: ./.github/actions/prep-env-and-cache - - name: Build - run: ./build.sh ${{ matrix.component }} + run: ./build.sh --save-image --extract-tarball ${{ matrix.component }} shell: bash - name: Upload results @@ -58,13 +55,6 @@ jobs: with: component: ${{ matrix.component }} - # TODO: remove extra step with https://phabricator.wikimedia.org/T347000 - - name: Upload additional bundle results for wikibase build - if: matrix.component == 'wikibase' - uses: ./.github/actions/upload-results - with: - component: ${{ matrix.component }}-bundle - - name: Scan Image uses: ./.github/actions/scan-image continue-on-error: true @@ -213,7 +203,7 @@ jobs: ] env: - TARGET_WIKIBASE_UPGRADE_IMAGE_NAME: wikibase-bundle + TARGET_WIKIBASE_UPGRADE_IMAGE_NAME: wikibase/wikibase-bundle steps: - uses: actions/checkout@v4 @@ -242,36 +232,6 @@ jobs: name: TestResults path: test/suites/**/results - # TODO: https://phabricator.wikimedia.org/T347084 - metadata: - runs-on: ubuntu-latest - timeout-minutes: 20 - - needs: - - test - - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@v4 - - - uses: c-py/action-dotenv-to-setenv@v4 - with: - env-file: "variables.env" - - - name: Build version requirements - shell: bash - run: | - ./build.sh requirements - - - name: Archive metadata artifacts - uses: actions/upload-artifact@v3 - with: - name: Metadata - path: artifacts/built_versions.log - if-no-files-found: error - upload_ghcr: runs-on: ubuntu-latest timeout-minutes: 20 diff --git a/.gitignore b/.gitignore index d0de669e7..c91374241 100644 --- a/.gitignore +++ b/.gitignore @@ -1,24 +1,10 @@ -core -Vector -cache -git_cache -container_id *.swp -Wikibase.tar.gz -*.docker.tar.gz - -Docker/build/Wikibase/artifacts - -Docker/build/WDQS/wait-for-it.sh - -#temp -Docker/**/*.tar.gz -test/suites/**/results/ -test/log -test/mwlog -docs/diagrams/node_modules/ -artifacts/* -local.env -example/log .DS_Store .direnv +artifacts/* +docs/diagrams/node_modules/ +example/log +local.env +test/log +test/mwlog +test/suites/**/results/ diff --git a/.nvmrc b/.nvmrc deleted file mode 100644 index 4a1f488b6..000000000 --- a/.nvmrc +++ /dev/null @@ -1 +0,0 @@ -18.17.1 diff --git a/Docker/build/Elasticsearch/Dockerfile b/Docker/build/Elasticsearch/Dockerfile deleted file mode 100644 index 170a95f9e..000000000 --- a/Docker/build/Elasticsearch/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -ARG ELASTICSEARCH_VERSION -FROM docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION} - -ARG ELASTICSEARCH_PLUGIN_EXTRA_VERSION -RUN ./bin/elasticsearch-plugin install org.wikimedia.search:extra:${ELASTICSEARCH_PLUGIN_EXTRA_VERSION} - -ARG ELASTICSEARCH_VERSION -RUN ./bin/elasticsearch-plugin install org.wikimedia.search.highlighter:experimental-highlighter-elasticsearch-plugin:${ELASTICSEARCH_VERSION} - -COPY default.jvm.options /default.jvm.options -RUN cat /default.jvm.options >> /usr/share/elasticsearch/config/jvm.options && rm /default.jvm.options - -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/QuickStatements/Dockerfile b/Docker/build/QuickStatements/Dockerfile deleted file mode 100644 index f6efa9492..000000000 --- a/Docker/build/QuickStatements/Dockerfile +++ /dev/null @@ -1,46 +0,0 @@ -ARG COMPOSER_IMAGE_NAME -ARG COMPOSER_IMAGE_VERSION -FROM ubuntu:xenial as fetcher - -COPY artifacts/quickstatements.tar.gz artifacts/quickstatements.tar.gz -RUN tar xzf artifacts/quickstatements.tar.gz - -FROM ${COMPOSER_IMAGE_NAME}:${COMPOSER_IMAGE_VERSION} as composer -COPY --from=fetcher --chown=nobody:nogroup /quickstatements/composer.json /quickstatements/composer.json - -WORKDIR /quickstatements -RUN composer install --no-dev - -FROM php:8.1-apache - -# Install envsubst -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends gettext-base jq libicu-dev && \ - rm -rf /var/lib/apt/lists/* && \ - docker-php-ext-configure intl && \ - docker-php-ext-install intl - -COPY --from=fetcher /quickstatements /var/www/html/quickstatements -COPY --from=composer --chown=root:root /quickstatements/vendor /var/www/html/quickstatements/vendor -COPY --from=fetcher /magnustools /var/www/html/magnustools - -COPY entrypoint.sh /entrypoint.sh - -COPY config.json /templates/config.json -COPY oauth.ini /templates/oauth.ini -COPY php.ini /templates/php.ini - -ENV APACHE_DOCUMENT_ROOT /var/www/html/quickstatements/public_html -RUN sed -ri -e "s!/var/www/html!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/sites-available/*.conf && \ - sed -ri -e "s!/var/www/!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf - -ENV MW_SITE_NAME=wikibase-docker\ - MW_SITE_LANG=en\ - PHP_TIMEZONE=UTC - -RUN install -d -owww-data /var/log/quickstatements - -ENTRYPOINT ["/bin/bash"] -CMD ["/entrypoint.sh"] - -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/Wikibase/Dockerfile b/Docker/build/Wikibase/Dockerfile deleted file mode 100644 index d74cc6b23..000000000 --- a/Docker/build/Wikibase/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -# Build configuration -ARG COMPOSER_IMAGE_NAME -ARG COMPOSER_IMAGE_VERSION -ARG MEDIAWIKI_IMAGE_NAME -ARG MEDIAWIKI_IMAGE_VERSION - -FROM ubuntu:xenial as unpacker - -RUN mkdir artifacts - -COPY artifacts/wikibase.tar.gz artifacts -RUN tar xzf artifacts/wikibase.tar.gz - -FROM ${MEDIAWIKI_IMAGE_NAME}:${MEDIAWIKI_IMAGE_VERSION} as collector - -COPY --from=unpacker Wikibase /var/www/html/extensions/Wikibase - -RUN rm /var/www/html/extensions/Wikibase/vendor -rf - -FROM ${COMPOSER_IMAGE_NAME}:${COMPOSER_IMAGE_VERSION} as composer -COPY --from=collector --chown=nobody:nogroup /var/www/html /var/www/html -WORKDIR /var/www/html/ -COPY composer.local.json /var/www/html/composer.local.json -RUN composer install --verbose -n --no-dev - -FROM ${MEDIAWIKI_IMAGE_NAME}:${MEDIAWIKI_IMAGE_VERSION} - -# Set error_reporting PHP.ini settings -# This is needed with PHP8+ and MediaWiki 1.39, as Wikibase contains deprecated code -RUN { \ - echo 'error_reporting = E_ALL ^ E_DEPRECATED'; \ - } > /usr/local/etc/php/conf.d/error_reporting.ini - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends libbz2-dev=1.* gettext-base && \ - rm -rf /var/lib/apt/lists/* && \ - a2enmod rewrite && \ - install -d /var/log/mediawiki -o www-data && \ - docker-php-ext-install calendar bz2 - -COPY --from=composer --chown=root:root /var/www/html /var/www/html -COPY artifacts/wait-for-it.sh /wait-for-it.sh -COPY entrypoint.sh /entrypoint.sh - -ARG MEDIAWIKI_SETTINGS_TEMPLATE_FILE -COPY ${MEDIAWIKI_SETTINGS_TEMPLATE_FILE} /LocalSettings.php.template -COPY htaccess /var/www/html/.htaccess - -RUN ln -s /var/www/html/ /var/www/html/w - -# Mediawiki configuration -ARG MW_SITE_NAME -ARG MW_SITE_LANG -ARG MW_WG_JOB_RUN_RATE -ARG MW_WG_ENABLE_UPLOADS -ARG MW_WG_UPLOAD_DIRECTORY -ARG WIKIBASE_PINGBACK - -ENV MW_SITE_NAME=${MW_SITE_NAME}\ - MW_SITE_LANG=${MW_SITE_LANG}\ - MW_WG_JOB_RUN_RATE=${MW_WG_JOB_RUN_RATE}\ - MW_WG_ENABLE_UPLOADS=${MW_WG_ENABLE_UPLOADS}\ - MW_WG_UPLOAD_DIRECTORY=${MW_WG_UPLOAD_DIRECTORY}\ - WIKIBASE_PINGBACK=${WIKIBASE_PINGBACK} - -RUN chown www-data ${MW_WG_UPLOAD_DIRECTORY} -R && \ - chmod o-w /var/www/html - -ENTRYPOINT ["/bin/bash"] -CMD ["/entrypoint.sh"] - -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/Wikibase/LocalSettings.php.REL1_35.template b/Docker/build/Wikibase/LocalSettings.php.REL1_35.template deleted file mode 100644 index 132e1d527..000000000 --- a/Docker/build/Wikibase/LocalSettings.php.REL1_35.template +++ /dev/null @@ -1,72 +0,0 @@ - '/var/log/mediawiki/resourceloader.log', - 'exception' => '/var/log/mediawiki/exception.log', - 'error' => '/var/log/mediawiki/error.log', - 'fatal' => '/var/log/mediawiki/fatal.log', -); - -## Site Settings -# TODO pass in the rest of this with env vars? -${DOLLAR}wgServer = WebRequest::detectServer(); -${DOLLAR}wgShellLocale = "en_US.utf8"; -${DOLLAR}wgLanguageCode = "${MW_SITE_LANG}"; -${DOLLAR}wgSitename = "${MW_SITE_NAME}"; -${DOLLAR}wgMetaNamespace = "Project"; -# Configured web paths & short URLs -# This allows use of the /wiki/* path -## https://www.mediawiki.org/wiki/Manual:Short_URL -${DOLLAR}wgScriptPath = "/w"; // this should already have been configured this way -${DOLLAR}wgArticlePath = "/wiki/${DOLLAR}1"; - -#Set Secret -${DOLLAR}wgSecretKey = "${MW_WG_SECRET_KEY}"; - -## RC Age -# https://www.mediawiki.org/wiki/Manual:$wgRCMaxAge -# Items in the recentchanges table are periodically purged; entries older than this many seconds will go. -# The query service (by default) loads data from recent changes -# Set this to 1 year to avoid any changes being removed from the RC table over a shorter period of time. -${DOLLAR}wgRCMaxAge = 365 * 24 * 3600; - -wfLoadSkin( 'Vector' ); - -// T276905 - default set in Dockerfile -${DOLLAR}wgJobRunRate = ${MW_WG_JOB_RUN_RATE}; - -${DOLLAR}wgEnableUploads = ${MW_WG_ENABLE_UPLOADS}; - -${DOLLAR}wgUploadDirectory = "${MW_WG_UPLOAD_DIRECTORY}"; - -## Wikibase -# Load Wikibase repo & client with the example / default settings. -require_once "${DOLLAR}IP/extensions/Wikibase/repo/Wikibase.php"; -require_once "${DOLLAR}IP/extensions/Wikibase/repo/ExampleSettings.php"; -require_once "${DOLLAR}IP/extensions/Wikibase/client/WikibaseClient.php"; -require_once "${DOLLAR}IP/extensions/Wikibase/client/ExampleSettings.php"; - -#Pingback -${DOLLAR}wgWBRepoSettings['wikibasePingback'] = ${WIKIBASE_PINGBACK}; - -foreach (glob("LocalSettings.d/*.php") as ${DOLLAR}filename) -{ - include ${DOLLAR}filename; -} diff --git a/Docker/build/Wikibase/default.env b/Docker/build/Wikibase/default.env deleted file mode 100644 index 1fd48b082..000000000 --- a/Docker/build/Wikibase/default.env +++ /dev/null @@ -1,6 +0,0 @@ -MW_SITE_NAME=wikibase-docker -MW_SITE_LANG=en -MW_WG_JOB_RUN_RATE=2 -MW_WG_ENABLE_UPLOADS=false -MW_WG_UPLOAD_DIRECTORY=/var/www/html/images -WIKIBASE_PINGBACK=false diff --git a/Docker/build/WikibaseBundle/Dockerfile b/Docker/build/WikibaseBundle/Dockerfile deleted file mode 100644 index 33d1d0ec9..000000000 --- a/Docker/build/WikibaseBundle/Dockerfile +++ /dev/null @@ -1,31 +0,0 @@ -ARG COMPOSER_IMAGE_NAME -ARG COMPOSER_IMAGE_VERSION -ARG WIKIBASE_IMAGE_NAME -FROM ${WIKIBASE_IMAGE_NAME}:latest as base - -FROM ${COMPOSER_IMAGE_NAME}:${COMPOSER_IMAGE_VERSION} as composer - -COPY --from=base --chown=nobody:nogroup /var/www/html /var/www/html -COPY artifacts/extensions /var/www/html/extensions - -WORKDIR /var/www/html/ -RUN rm -rf /var/www/html/vendor && \ - rm -rf /var/www/html/composer.lock && \ - composer install --no-dev -vv -n - -FROM ${WIKIBASE_IMAGE_NAME}:latest -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends lua5.1 && \ - rm -rf /var/lib/apt/lists/* && \ - rm -rf /var/www/html/vendor -COPY --from=composer /var/www/html /var/www/html -COPY extra-install/ /extra-install/ -COPY artifacts/oauth.ini /templates/oauth.ini - -# MW_WG_UPLOAD_DIRECTORY set in base image -RUN chown www-data ${MW_WG_UPLOAD_DIRECTORY} -R - -# copy extension settings -COPY LocalSettings.d /var/www/html/LocalSettings.d - -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 47bfbec54..000000000 --- a/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -FROM docker:latest - -RUN apk add --no-cache git make bash python3 py3-pip && \ - pip3 install pyyaml - -WORKDIR /app - -ENV XDG_CACHE_HOME=/app/cache -ENV HOME=/tmp - -ADD .github/workflows/build_and_test.yml config.yml diff --git a/Makefile b/Makefile index d955c2697..068273e81 100644 --- a/Makefile +++ b/Makefile @@ -62,39 +62,7 @@ endif @bash test/scripts/test_suite.sh base__fedprops @bash test/scripts/test_suite.sh base__pingback -# TODO: https://phabricator.wikimedia.org/T347084 -requirements: - python3 build/requirements/build_version_requirements.py - cat artifacts/built_versions.log - -wikibase: - bash update_cache.sh bundle - eval ". ./build/build_wikibase.sh; bash build/build_wikibase_docker.sh ${WIKIBASE_IMAGE_NAME};bash build/build_wikibase_bundle_docker.sh ${WIKIBASE_IMAGE_NAME} ${WIKIBASE_BUNDLE_IMAGE_NAME}" - -quickstatements: - bash update_cache.sh services - eval ". ./build/build_quickstatements.sh; bash build/build_quickstatements_docker.sh ${QUICKSTATEMENTS_IMAGE_NAME}" - -wdqs: - eval ". ./build/build_wdqs.sh; bash build/build_wdqs_docker.sh ${WDQS_IMAGE_NAME}" - -wdqs-proxy: - bash build/build_wdqs_proxy_docker.sh ${WDQS_PROXY_IMAGE_NAME} - -wdqs-frontend: - bash update_cache.sh services - eval ". ./build/build_wdqs_frontend.sh; bash build/build_wdqs_frontend_docker.sh ${WDQS_FRONTEND_IMAGE_NAME}" - -elasticsearch: - bash build/build_elasticsearch_docker.sh ${ELASTICSEARCH_IMAGE_NAME} - -clean-cache: - rm -rf cache/* - rm -rf git_cache/* - clean: rm -rf artifacts/*.tar.gz rm -rf artifacts/*.log rm -rf artifacts/*.env - -all: wikibase elasticsearch wdqs wdqs-frontend wdqs-proxy quickstatements diff --git a/README.md b/README.md index eba546748..dfbab8459 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ The wikibase release pipeline contains scripts used for building, testing and pu It contains a set of build targets defined in the [Makefile](./Makefile) which can be executed in two different ways. -When [building](docs/topics/pipeline.md), use the [build.sh](build.sh) wrapper script to do the building inside a docker-container, this way files aren't littered around the repository but only in the `artifacts/` directory. +When [building](docs/topics/pipeline.md), use the [build.sh](build.sh) script. For [testing](docs/topics/testing.md), you can use `test`, `test-all` make targets. @@ -15,8 +15,20 @@ For [testing](docs/topics/testing.md), you can use `test`, `test-all` make targe ### Build Commands ``` -$ ./build.sh all +# Build all wikibase suite components docker images +$ ./build.sh + +# Build only the mediawiki/wikibase containers $ ./build.sh wikibase + +# Build only the query service container and save the docker image to a tarball +$ ./build.sh --save-image wdqs + +# Build the wdqs-frontend container and extract a standalone tarball from the webroot +$ ./build.sh --extract-tarball wdqs-frontend + +# Build the wdqs container without using Dockers cache +$ ./build.sh --no-cache wdqs ``` ### Test Commands diff --git a/build.sh b/build.sh index be12ef6da..0c8913aef 100755 --- a/build.sh +++ b/build.sh @@ -1,63 +1,277 @@ #!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +# set -x -set -e +set -o allexport +# shellcheck disable=SC1091 +source ./variables.env +if [ -f ./local.env ]; then + # shellcheck disable=SC1091 + source ./local.env +fi +set +o allexport -TARGET=$1 -if ! [[ -d .git ]]; then - echo "ERROR: builder.sh must be executed from the root of the repository!" >&2 - exit 1 -fi +SAVE_IMAGE=false +EXTRACT_TARBALL=false +DOCKER_BUILD_CACHE_OPT="" -usage() { - echo "USAGE: $0 " >&2 +function save_image { + local image_name="$1" + local image_url="$2" + local image_name_with_tag="$3" + local image_url_with_tag="$4" + + if $SAVE_IMAGE; then + docker save "$image_url" "$image_url_with_tag" | \ + gzip -"$GZIP_COMPRESSION_RATE" > "artifacts/${image_name_with_tag//:/-}.docker.tar.gz" + pushd artifacts + ln -sf "${image_name_with_tag//:/-}.docker.tar.gz" "${image_name}.docker.tar.gz" + popd + fi } -if [[ -z "$TARGET" ]]; then - echo "ERROR: TARGET is not set!" >&2 - usage - exit 1 -fi -if ! [[ -f "local.env" ]]; then - touch local.env -fi +# wikibase/wdqs -> wdqs +function image_url_to_image_name { + local image_url="$1" + + # this will complain about image urls with tags in the end, e.g. foo:latest + colon_count=$(grep -o ":" <<< "$image_url" | wc -l) + if [[ $colon_count -gt 0 ]]; then + echo "Incompatible image url '${image_url}'. Only dockerhub 'namespace/image_name' supported." > /dev/stderr + exit 1 + fi + + # this will complain about image urls with hostnames, e.g. ghcr.io/foo/bar + slash_count=$(grep -o "/" <<< "$image_url" | wc -l) + if [[ $slash_count -gt 1 ]]; then + echo "Incompatible image url '${image_url}'. Only dockerhub 'namespace/image_name' supported." > /dev/stderr + exit 1 + fi + + # only take the second half of the url, the name after the slash + echo "$image_url" | cut -d'/' -f 2 +} + + +function setup_image_name_url_and_tag { + local version_string="$2" + + # ‼️ HEADS UP! This will set the global vars 👿 + image_url="$1" + image_name="$(image_url_to_image_name "$image_url")" + image_name_with_tag="${image_name}:${version_string}" + image_url_with_tag="${image_url}:${version_string}" +} + + +function build_wikibase { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg COMPOSER_IMAGE_URL="$COMPOSER_IMAGE_URL" \ + --build-arg MEDIAWIKI_IMAGE_URL="$MEDIAWIKI_IMAGE_URL" \ + --build-arg WIKIBASE_COMMIT="$WIKIBASE_COMMIT" \ + \ + --build-arg MW_SITE_NAME="$MW_SITE_NAME" \ + --build-arg MW_SITE_LANG="$MW_SITE_LANG" \ + --build-arg MW_WG_JOB_RUN_RATE="$MW_WG_JOB_RUN_RATE" \ + --build-arg MW_WG_ENABLE_UPLOADS="$MW_WG_ENABLE_UPLOADS" \ + --build-arg MW_WG_UPLOAD_DIRECTORY="$MW_WG_UPLOAD_DIRECTORY" \ + --build-arg WIKIBASE_PINGBACK="$WIKIBASE_PINGBACK" \ + \ + build/Wikibase -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" + + if $EXTRACT_TARBALL; then + docker run --entrypoint="" --rm "$image_url_with_tag" \ + tar cz -C /var/www --transform="s,^html,${image_name}," html \ + > "artifacts/${image_name_with_tag//:/-}.tar.gz" + pushd artifacts + ln -sf "${image_name_with_tag//:/-}.tar.gz" "${image_name}.tar.gz" + popd + fi + + setup_image_name_url_and_tag "$WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg COMPOSER_IMAGE_URL="$COMPOSER_IMAGE_URL" \ + --build-arg WMDE_RELEASE_VERSION="$WMDE_RELEASE_VERSION" \ + --build-arg RELEASE_VERSION="$RELEASE_VERSION" \ + --build-arg WIKIBASE_SUITE_WIKIBASE_IMAGE_URL="$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" \ + \ + --build-arg BABEL_COMMIT="$BABEL_COMMIT" \ + --build-arg CLDR_COMMIT="$CLDR_COMMIT" \ + --build-arg BABEL_COMMIT="$BABEL_COMMIT" \ + --build-arg CLDR_COMMIT="$CLDR_COMMIT" \ + --build-arg CIRRUSSEARCH_COMMIT="$CIRRUSSEARCH_COMMIT" \ + --build-arg CONFIRMEDIT_COMMIT="$CONFIRMEDIT_COMMIT" \ + --build-arg ELASTICA_COMMIT="$ELASTICA_COMMIT" \ + --build-arg ENTITYSCHEMA_COMMIT="$ENTITYSCHEMA_COMMIT" \ + --build-arg NUKE_COMMIT="$NUKE_COMMIT" \ + --build-arg OAUTH_COMMIT="$OAUTH_COMMIT" \ + --build-arg SCRIBUNTO_COMMIT="$SCRIBUNTO_COMMIT" \ + --build-arg SYNTAXHIGHLIGHT_GESHI_COMMIT="$SYNTAXHIGHLIGHT_GESHI_COMMIT" \ + --build-arg UNIVERSALLANGUAGESELECTOR_COMMIT="$UNIVERSALLANGUAGESELECTOR_COMMIT" \ + --build-arg VISUALEDITOR_COMMIT="$VISUALEDITOR_COMMIT" \ + --build-arg WIKIBASECIRRUSSEARCH_COMMIT="$WIKIBASECIRRUSSEARCH_COMMIT" \ + --build-arg WIKIBASEMANIFEST_COMMIT="$WIKIBASEMANIFEST_COMMIT" \ + --build-arg WIKIBASEEDTF_COMMIT="$WIKIBASEEDTF_COMMIT" \ + --build-arg WIKIBASELOCALMEDIA_COMMIT="$WIKIBASELOCALMEDIA_COMMIT" \ + \ + build/WikibaseBundle -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" +} + + +function build_elasticseach { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL" "$ELASTICSEARCH_VERSION-$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg=ELASTICSEARCH_IMAGE_URL="$ELASTICSEARCH_IMAGE_URL" \ + --build-arg=ELASTICSEARCH_PLUGIN_WIKIMEDIA_EXTRA="$ELASTICSEARCH_PLUGIN_WIKIMEDIA_EXTRA" \ + --build-arg=ELASTICSEARCH_PLUGIN_WIKIMEDIA_HIGHLIGHTER="$ELASTICSEARCH_PLUGIN_WIKIMEDIA_HIGHLIGHTER" \ + \ + build/Elasticsearch/ -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" +} + -HOST_TMP="$(mktemp -d)" - -remove_builder_tmp() { - rm -rf "${HOST_TMP}" -} -trap remove_builder_tmp EXIT - -docker build --quiet . -t builder:latest - -docker run \ - --rm \ - \ - --user "$(id -u):$(id -g)" \ - --group-add "$(getent group docker | cut -d: -f3)" \ - \ - -v "$(pwd)/Makefile":/app/Makefile \ - -v "$(pwd)/local.env":/app/local.env \ - -v "$(pwd)/variables.env":/app/variables.env \ - \ - -v "$(pwd)/build":/app/build \ - -v "$(pwd)/Docker/build":/app/Docker/build \ - \ - -v "$(pwd)/update_cache.sh":/app/update_cache.sh \ - -v "$(pwd)/cache":/app/cache \ - -v "$(pwd)/git_cache":/app/git_cache \ - \ - -v "$(pwd)/artifacts":/app/artifacts \ - \ - -v /var/run/docker.sock:/var/run/docker.sock \ - \ - -v "${HOST_TMP}":/tmp \ - -e HOST_TMP="${HOST_TMP}" \ - \ - -e BUILD_TIMESTAMP="$(date +%Y%m%d%H%M%S)" \ - -e GIT_REVISION_HASH="$(git rev-parse --short HEAD)" \ - -e GIT_REVISION_BRANCH="$(git rev-parse --abbrev-ref HEAD)" \ - \ - builder:latest make "$TARGET" +function build_wdqs { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_WDQS_IMAGE_URL" "$WDQS_VERSION-$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg DEBIAN_IMAGE_URL="$DEBIAN_IMAGE_URL" \ + --build-arg JDK_IMAGE_URL="$JDK_IMAGE_URL" \ + --build-arg WDQS_VERSION="$WDQS_VERSION" \ + \ + build/WDQS/ -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" +} + + +function build_wdqs-frontend { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL" "$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg COMPOSER_IMAGE_URL="$COMPOSER_IMAGE_URL" \ + --build-arg NGINX_IMAGE_URL="$NGINX_IMAGE_URL" \ + --build-arg NODE_IMAGE_URL="$NODE_IMAGE_URL" \ + --build-arg WDQSQUERYGUI_COMMIT="$WDQSQUERYGUI_COMMIT" \ + \ + build/WDQS-frontend/ -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" + + if $EXTRACT_TARBALL; then + docker run --entrypoint="" --rm "$image_url_with_tag" \ + tar cz -C /usr/share/nginx --transform="s,^html,${image_name}," html \ + > "artifacts/${image_name_with_tag//:/-}.tar.gz" + pushd artifacts + ln -sf "${image_name_with_tag//:/-}.tar.gz" "${image_name}.tar.gz" + popd + fi +} + + +function build_wdqs-proxy { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL" "$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg NGINX_IMAGE_URL="$NGINX_IMAGE_URL" \ + \ + build/WDQS-proxy/ -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" +} + + +function build_quickstatements { + setup_image_name_url_and_tag "$WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL" "$WMDE_RELEASE_VERSION" + + docker build \ + $DOCKER_BUILD_CACHE_OPT \ + --build-arg COMPOSER_IMAGE_URL="$COMPOSER_IMAGE_URL" \ + --build-arg PHP_IMAGE_URL="$PHP_IMAGE_URL" \ + --build-arg QUICKSTATEMENTS_COMMIT="$QUICKSTATEMENTS_COMMIT" \ + --build-arg MAGNUSTOOLS_COMMIT="$MAGNUSTOOLS_COMMIT" \ + \ + build/QuickStatements/ -t "$image_url_with_tag" -t "$image_url" + + save_image "$image_name" "$image_url" "$image_name_with_tag" "$image_url_with_tag" +} + + +function build_all { + build_wikibase + build_elasticseach + build_wdqs + build_wdqs-frontend + build_wdqs-proxy + build_quickstatements +} + + +build_target_set=false + +for arg in "$@"; do + case $arg in + wikibase) + build_wikibase + build_target_set=true + ;; + elasticsearch) + build_elasticseach + build_target_set=true + ;; + wdqs) + build_wdqs + build_target_set=true + ;; + wdqs-frontend) + build_wdqs-frontend + build_target_set=true + ;; + wdqs-proxy) + build_wdqs-proxy + build_target_set=true + ;; + quickstatements) + build_quickstatements + build_target_set=true + ;; + all) + build_all + build_target_set=true + ;; + -s|--save-image) + SAVE_IMAGE=true + ;; + -t|--extract-tarball) + EXTRACT_TARBALL=true + ;; + -n|--no-cache) + DOCKER_BUILD_CACHE_OPT="--no-cache" + ;; + *) + echo "Unknown argument: $arg" > /dev/stderr + exit 1 + ;; + esac +done + +if ! $build_target_set; then + build_all +fi diff --git a/build/Elasticsearch/Dockerfile b/build/Elasticsearch/Dockerfile new file mode 100644 index 000000000..178160cfc --- /dev/null +++ b/build/Elasticsearch/Dockerfile @@ -0,0 +1,18 @@ +ARG ELASTICSEARCH_IMAGE_URL + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${ELASTICSEARCH_IMAGE_URL} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" +ARG ELASTICSEARCH_PLUGIN_WIKIMEDIA_EXTRA +ARG ELASTICSEARCH_PLUGIN_WIKIMEDIA_HIGHLIGHTER + +RUN \ + ./bin/elasticsearch-plugin install \ + org.wikimedia.search:extra:${ELASTICSEARCH_PLUGIN_WIKIMEDIA_EXTRA} && \ + ./bin/elasticsearch-plugin install \ + org.wikimedia.search.highlighter:experimental-highlighter-elasticsearch-plugin:${ELASTICSEARCH_PLUGIN_WIKIMEDIA_HIGHLIGHTER} +COPY default.jvm.options /default.jvm.options +RUN cat /default.jvm.options >> /usr/share/elasticsearch/config/jvm.options && \ + rm /default.jvm.options + diff --git a/Docker/build/Elasticsearch/README.md b/build/Elasticsearch/README.md similarity index 100% rename from Docker/build/Elasticsearch/README.md rename to build/Elasticsearch/README.md diff --git a/Docker/build/Elasticsearch/default.jvm.options b/build/Elasticsearch/default.jvm.options similarity index 100% rename from Docker/build/Elasticsearch/default.jvm.options rename to build/Elasticsearch/default.jvm.options diff --git a/build/QuickStatements/Dockerfile b/build/QuickStatements/Dockerfile new file mode 100644 index 000000000..3629208b0 --- /dev/null +++ b/build/QuickStatements/Dockerfile @@ -0,0 +1,53 @@ +ARG COMPOSER_IMAGE_URL +ARG PHP_IMAGE_URL + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as composer +ARG QUICKSTATEMENTS_COMMIT +ARG MAGNUSTOOLS_COMMIT + +RUN \ + git clone https://github.com/magnusmanske/quickstatements /tmp/quickstatements && \ + git -C /tmp/quickstatements checkout ${QUICKSTATEMENTS_COMMIT} && \ + git -C /tmp/quickstatements submodule update --init --recursive && \ + \ + git clone https://bitbucket.org/magnusmanske/magnustools.git /tmp/magnustools && \ + git -C /tmp/magnustools checkout ${MAGNUSTOOLS_COMMIT} + +WORKDIR /tmp/quickstatements +RUN composer install --no-dev + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${PHP_IMAGE_URL} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install --yes --no-install-recommends gettext-base jq libicu-dev && \ + rm -rf /var/lib/apt/lists/* && \ + \ + docker-php-ext-configure intl && \ + docker-php-ext-install intl + +COPY --from=composer /tmp/quickstatements /var/www/html/quickstatements +COPY --from=composer /tmp/magnustools /var/www/html/magnustools + +COPY entrypoint.sh /entrypoint.sh + +COPY config.json /templates/config.json +COPY oauth.ini /templates/oauth.ini +COPY php.ini /templates/php.ini + +ENV APACHE_DOCUMENT_ROOT /var/www/html/quickstatements/public_html +RUN sed -ri -e "s!/var/www/html!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/sites-available/*.conf && \ + sed -ri -e "s!/var/www/!${APACHE_DOCUMENT_ROOT}!g" /etc/apache2/apache2.conf /etc/apache2/conf-available/*.conf + +ENV MW_SITE_NAME=wikibase-docker \ + MW_SITE_LANG=en \ + PHP_TIMEZONE=UTC + +RUN install -d -owww-data /var/log/quickstatements + +ENTRYPOINT ["/bin/bash"] +CMD ["/entrypoint.sh"] diff --git a/Docker/build/QuickStatements/README.md b/build/QuickStatements/README.md similarity index 100% rename from Docker/build/QuickStatements/README.md rename to build/QuickStatements/README.md diff --git a/Docker/build/QuickStatements/config.json b/build/QuickStatements/config.json similarity index 100% rename from Docker/build/QuickStatements/config.json rename to build/QuickStatements/config.json diff --git a/Docker/build/QuickStatements/entrypoint.sh b/build/QuickStatements/entrypoint.sh similarity index 100% rename from Docker/build/QuickStatements/entrypoint.sh rename to build/QuickStatements/entrypoint.sh diff --git a/Docker/build/QuickStatements/oauth.ini b/build/QuickStatements/oauth.ini similarity index 100% rename from Docker/build/QuickStatements/oauth.ini rename to build/QuickStatements/oauth.ini diff --git a/Docker/build/QuickStatements/php.ini b/build/QuickStatements/php.ini similarity index 100% rename from Docker/build/QuickStatements/php.ini rename to build/QuickStatements/php.ini diff --git a/Docker/build/WDQS-frontend/Dockerfile b/build/WDQS-frontend/Dockerfile similarity index 52% rename from Docker/build/WDQS-frontend/Dockerfile rename to build/WDQS-frontend/Dockerfile index 2d4d8a1cb..4f941c1a2 100644 --- a/Docker/build/WDQS-frontend/Dockerfile +++ b/build/WDQS-frontend/Dockerfile @@ -1,14 +1,24 @@ -FROM ubuntu:xenial as fetcher +ARG COMPOSER_IMAGE_URL +ARG NGINX_IMAGE_URL +ARG NODE_IMAGE_URL -ARG tarball -COPY $tarball . -RUN mkdir wikidata-query-gui \ - && tar xfv $tarball -C ./wikidata-query-gui \ - && rm $tarball +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as composer +ARG WDQSQUERYGUI_COMMIT -FROM node:14-alpine as builder +RUN git clone https://gerrit.wikimedia.org/r/wikidata/query/gui /tmp/wikidata-query-gui && \ + git -C /tmp/wikidata-query-gui checkout ${WDQSQUERYGUI_COMMIT} && \ + git -C /tmp/wikidata-query-gui submodule update --init --recursive && \ + rm -f /tmp/wikidata-query-gui/.travis.yml && \ + find /tmp/wikidata-query-gui -name ".git*" -exec rm -rf {} + -COPY --from=fetcher /wikidata-query-gui /tmp/wikidata-query-gui + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${NODE_IMAGE_URL} as builder + +COPY --from=composer /tmp/wikidata-query-gui /tmp/wikidata-query-gui WORKDIR /tmp/wikidata-query-gui @@ -26,7 +36,10 @@ RUN mv package.json package.json.orig \ && npm run build -FROM nginx:stable-alpine as final +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${NGINX_IMAGE_URL} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" WORKDIR /usr/share/nginx/html @@ -43,4 +56,3 @@ ENV LANGUAGE=en\ ENTRYPOINT ["/entrypoint.sh"] CMD ["nginx", "-g", "daemon off;"] -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/WDQS-frontend/README.md b/build/WDQS-frontend/README.md similarity index 100% rename from Docker/build/WDQS-frontend/README.md rename to build/WDQS-frontend/README.md diff --git a/Docker/build/WDQS-frontend/custom-config.json b/build/WDQS-frontend/custom-config.json similarity index 100% rename from Docker/build/WDQS-frontend/custom-config.json rename to build/WDQS-frontend/custom-config.json diff --git a/Docker/build/WDQS-frontend/default.conf b/build/WDQS-frontend/default.conf similarity index 100% rename from Docker/build/WDQS-frontend/default.conf rename to build/WDQS-frontend/default.conf diff --git a/Docker/build/WDQS-frontend/entrypoint.sh b/build/WDQS-frontend/entrypoint.sh similarity index 100% rename from Docker/build/WDQS-frontend/entrypoint.sh rename to build/WDQS-frontend/entrypoint.sh diff --git a/Docker/build/WDQS-proxy/Dockerfile b/build/WDQS-proxy/Dockerfile similarity index 61% rename from Docker/build/WDQS-proxy/Dockerfile rename to build/WDQS-proxy/Dockerfile index a89ad84e1..e10563aa5 100644 --- a/Docker/build/WDQS-proxy/Dockerfile +++ b/build/WDQS-proxy/Dockerfile @@ -1,4 +1,9 @@ -FROM nginx:stable-alpine +ARG NGINX_IMAGE_URL + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${NGINX_IMAGE_URL} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" COPY entrypoint.sh /entrypoint.sh COPY wdqs.template /etc/nginx/conf.d/wdqs.template @@ -7,4 +12,3 @@ ENV PROXY_MAX_QUERY_MILLIS=60000 ENTRYPOINT "/entrypoint.sh" -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/WDQS-proxy/README.md b/build/WDQS-proxy/README.md similarity index 100% rename from Docker/build/WDQS-proxy/README.md rename to build/WDQS-proxy/README.md diff --git a/Docker/build/WDQS-proxy/entrypoint.sh b/build/WDQS-proxy/entrypoint.sh similarity index 100% rename from Docker/build/WDQS-proxy/entrypoint.sh rename to build/WDQS-proxy/entrypoint.sh diff --git a/Docker/build/WDQS-proxy/wdqs.template b/build/WDQS-proxy/wdqs.template similarity index 100% rename from Docker/build/WDQS-proxy/wdqs.template rename to build/WDQS-proxy/wdqs.template diff --git a/Docker/build/WDQS/Dockerfile b/build/WDQS/Dockerfile similarity index 54% rename from Docker/build/WDQS/Dockerfile rename to build/WDQS/Dockerfile index 5edc31f82..6484fdcc8 100644 --- a/Docker/build/WDQS/Dockerfile +++ b/build/WDQS/Dockerfile @@ -1,16 +1,30 @@ -FROM ubuntu:xenial as fetcher +ARG DEBIAN_IMAGE_URL +ARG JDK_IMAGE_URL -RUN apt-get update && \ - apt-get install --yes --no-install-recommends unzip=6.* && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -ARG tarball -COPY $tarball . -RUN mkdir wdqs-service && \ - tar xfv $tarball -C ./wdqs-service --strip-components=1 \ - && rm $tarball +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${DEBIAN_IMAGE_URL} as fetcher +ARG WDQS_VERSION -FROM openjdk:8-jdk-alpine +RUN apt-get update && \ + apt-get install --yes --no-install-recommends wget ca-certificates && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] +RUN set -x; \ + TARBALL="service-${WDQS_VERSION}-dist.tar.gz"; \ + wget --no-verbose "https://archiva.wikimedia.org/repository/releases/org/wikidata/query/rdf/service/$WDQS_VERSION/$TARBALL"; \ + SIGNATURE="$TARBALL".md5; \ + wget --no-verbose "https://archiva.wikimedia.org/repository/releases/org/wikidata/query/rdf/service/$WDQS_VERSION/$SIGNATURE"; \ + echo "$(cat $SIGNATURE) $TARBALL" | md5sum -c; \ + mkdir /tmp/wdqs-service; \ + tar zxvf $TARBALL -C /tmp/wdqs-service --strip-components=1 + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${JDK_IMAGE_URL} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" # Blazegraph scripts require bash # Install gettext for envsubst command, (it needs libintl package) @@ -19,7 +33,7 @@ RUN set -x ; \ apk --no-cache add bash=\<4.5.0 gettext=\<0.19.8.2 libintl=\<0.19.8.2 curl=\<7.64.999 su-exec=\~0.2 && \ addgroup -g 66 -S blazegraph && adduser -S -G blazegraph -u 666 -s /bin/bash blazegraph -COPY --from=fetcher --chown=blazegraph:blazegraph /wdqs-service /wdqs +COPY --from=fetcher --chown=blazegraph:blazegraph /tmp/wdqs-service /wdqs RUN mkdir /var/log/wdqs && chown blazegraph /var/log/wdqs @@ -43,5 +57,3 @@ COPY --chown=blazegraph:blazegraph RWStore.properties whitelist.txt logback.xml RUN chmod +x /wdqs/runUpdate.sh ENTRYPOINT ["/entrypoint.sh"] - -LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" diff --git a/Docker/build/WDQS/README.md b/build/WDQS/README.md similarity index 100% rename from Docker/build/WDQS/README.md rename to build/WDQS/README.md diff --git a/Docker/build/WDQS/RWStore.properties b/build/WDQS/RWStore.properties similarity index 100% rename from Docker/build/WDQS/RWStore.properties rename to build/WDQS/RWStore.properties diff --git a/Docker/build/WDQS/entrypoint.sh b/build/WDQS/entrypoint.sh similarity index 100% rename from Docker/build/WDQS/entrypoint.sh rename to build/WDQS/entrypoint.sh diff --git a/Docker/build/WDQS/logback.xml b/build/WDQS/logback.xml similarity index 100% rename from Docker/build/WDQS/logback.xml rename to build/WDQS/logback.xml diff --git a/Docker/build/WDQS/mwservices.json b/build/WDQS/mwservices.json similarity index 100% rename from Docker/build/WDQS/mwservices.json rename to build/WDQS/mwservices.json diff --git a/Docker/build/WDQS/runBlazegraph.sh b/build/WDQS/runBlazegraph.sh similarity index 100% rename from Docker/build/WDQS/runBlazegraph.sh rename to build/WDQS/runBlazegraph.sh diff --git a/Docker/build/WDQS/runUpdate.sh b/build/WDQS/runUpdate.sh similarity index 100% rename from Docker/build/WDQS/runUpdate.sh rename to build/WDQS/runUpdate.sh diff --git a/Docker/build/WDQS/wait-for-it.sh b/build/WDQS/wait-for-it.sh similarity index 100% rename from Docker/build/WDQS/wait-for-it.sh rename to build/WDQS/wait-for-it.sh diff --git a/Docker/build/WDQS/whitelist.txt b/build/WDQS/whitelist.txt similarity index 100% rename from Docker/build/WDQS/whitelist.txt rename to build/WDQS/whitelist.txt diff --git a/build/Wikibase/Dockerfile b/build/Wikibase/Dockerfile new file mode 100644 index 000000000..a0816b15d --- /dev/null +++ b/build/Wikibase/Dockerfile @@ -0,0 +1,80 @@ +# Build configuration +ARG MEDIAWIKI_IMAGE_URL +ARG COMPOSER_IMAGE_URL + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as git-checkout +ARG WIKIBASE_COMMIT +RUN git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Wikibase /tmp/wikibase && \ + git -C /tmp/wikibase checkout ${WIKIBASE_COMMIT} && \ + git -C /tmp/wikibase submodule update --init --recursive && \ + rm -f /tmp/wikibase/.travis.yml && \ + find /tmp/wikibase -name ".git*" -exec rm -rf {} + + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${MEDIAWIKI_IMAGE_URL} as collector +COPY --from=git-checkout /tmp/wikibase /var/www/html/extensions/Wikibase + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as composer +COPY --from=collector --chown=nobody:nogroup /var/www/html /var/www/html +WORKDIR /var/www/html/ +COPY composer.local.json /var/www/html/composer.local.json +RUN composer install --verbose -n --no-dev \ + --ignore-platform-req=ext-calendar \ + --ignore-platform-req=ext-intl + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${MEDIAWIKI_IMAGE_URL} +ARG GIT_CURRENT_REVISION +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" +# Set error_reporting PHP.ini settings +# This is needed with PHP8+ and MediaWiki 1.39, as Wikibase contains deprecated code +# TODO: remove this and see how far we get +RUN { \ + echo 'error_reporting = E_ALL ^ E_DEPRECATED'; \ +} > /usr/local/etc/php/conf.d/error_reporting.ini + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive \ + apt-get install --yes --no-install-recommends libbz2-dev=1.* gettext-base && \ + rm -rf /var/lib/apt/lists/* && \ + \ + a2enmod rewrite && \ + install -d /var/log/mediawiki -o www-data && \ + docker-php-ext-install calendar bz2 + +COPY --from=composer --chown=root:root /var/www/html /var/www/html +COPY wait-for-it.sh /wait-for-it.sh +COPY entrypoint.sh /entrypoint.sh + +COPY LocalSettings.php.template /LocalSettings.php.template +COPY htaccess /var/www/html/.htaccess + +RUN ln -s /var/www/html/ /var/www/html/w + +# Mediawiki build time configuration +ARG MW_SITE_NAME +ARG MW_SITE_LANG +ARG MW_WG_JOB_RUN_RATE +ARG MW_WG_ENABLE_UPLOADS +ARG MW_WG_UPLOAD_DIRECTORY +ARG WIKIBASE_PINGBACK + +ENV MW_SITE_NAME=${MW_SITE_NAME}\ + MW_SITE_LANG=${MW_SITE_LANG}\ + MW_WG_JOB_RUN_RATE=${MW_WG_JOB_RUN_RATE}\ + MW_WG_ENABLE_UPLOADS=${MW_WG_ENABLE_UPLOADS}\ + MW_WG_UPLOAD_DIRECTORY=${MW_WG_UPLOAD_DIRECTORY}\ + WIKIBASE_PINGBACK=${WIKIBASE_PINGBACK} + +RUN mkdir /var/www/html/uploads && \ + chown www-data /var/www/html/uploads -R && \ + chmod o-w /var/www/html + +ENTRYPOINT ["/bin/bash"] +CMD ["/entrypoint.sh"] diff --git a/Docker/build/Wikibase/LocalSettings.php.template b/build/Wikibase/LocalSettings.php.template similarity index 100% rename from Docker/build/Wikibase/LocalSettings.php.template rename to build/Wikibase/LocalSettings.php.template diff --git a/Docker/build/Wikibase/README.md b/build/Wikibase/README.md similarity index 100% rename from Docker/build/Wikibase/README.md rename to build/Wikibase/README.md diff --git a/Docker/build/Wikibase/composer.local.json b/build/Wikibase/composer.local.json similarity index 100% rename from Docker/build/Wikibase/composer.local.json rename to build/Wikibase/composer.local.json diff --git a/Docker/build/Wikibase/entrypoint.sh b/build/Wikibase/entrypoint.sh similarity index 100% rename from Docker/build/Wikibase/entrypoint.sh rename to build/Wikibase/entrypoint.sh diff --git a/Docker/build/Wikibase/htaccess b/build/Wikibase/htaccess similarity index 100% rename from Docker/build/Wikibase/htaccess rename to build/Wikibase/htaccess diff --git a/Docker/build/wait-for-it.sh b/build/Wikibase/wait-for-it.sh similarity index 100% rename from Docker/build/wait-for-it.sh rename to build/Wikibase/wait-for-it.sh diff --git a/build/WikibaseBundle/Dockerfile b/build/WikibaseBundle/Dockerfile new file mode 100644 index 000000000..bca09e1d4 --- /dev/null +++ b/build/WikibaseBundle/Dockerfile @@ -0,0 +1,80 @@ +ARG COMPOSER_IMAGE_URL +ARG WMDE_RELEASE_VERSION +ARG RELEASE_VERSION +ARG WIKIBASE_SUITE_WIKIBASE_IMAGE_URL + +# ########################################################################### +FROM ${WIKIBASE_SUITE_WIKIBASE_IMAGE_URL}:${RELEASE_VERSION}-${WMDE_RELEASE_VERSION} as wikibase + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as composer + +COPY --from=wikibase --chown=nobody:nogroup /var/www/html /var/www/html +WORKDIR /var/www/html + +COPY LocalSettings.d.template /tmp/LocalSettings.d.template + +ARG ALL_EXTENSIONS="Babel,cldr,CirrusSearch,ConfirmEdit,Elastica,EntitySchema,Nuke,OAuth,Scribunto,SyntaxHighlight_GeSHi,UniversalLanguageSelector,VisualEditor,WikibaseCirrusSearch,WikibaseManifest" +ARG BABEL_COMMIT +ARG CLDR_COMMIT +ARG CIRRUSSEARCH_COMMIT +ARG CONFIRMEDIT_COMMIT +ARG ELASTICA_COMMIT +ARG ENTITYSCHEMA_COMMIT +ARG NUKE_COMMIT +ARG OAUTH_COMMIT +ARG SCRIBUNTO_COMMIT +ARG SYNTAXHIGHLIGHT_GESHI_COMMIT +ARG UNIVERSALLANGUAGESELECTOR_COMMIT +ARG VISUALEDITOR_COMMIT +ARG WIKIBASECIRRUSSEARCH_COMMIT +ARG WIKIBASEMANIFEST_COMMIT +ARG WIKIBASEEDTF_COMMIT +ARG WIKIBASELOCALMEDIA_COMMIT + +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] +RUN set -x; \ + IFS=',' read -ra EXTENSIONS <<< ${ALL_EXTENSIONS}; \ + mkdir LocalSettings.d; \ + for EXTENSION in "${EXTENSIONS[@]}"; do \ + rm -rf "extensions/${EXTENSION}"; \ + git clone "https://gerrit.wikimedia.org/r/mediawiki/extensions/${EXTENSION}" "extensions/${EXTENSION}"; \ + EXTENSION_COMMIT_VAR="${EXTENSION^^}_COMMIT"; \ + EXTENSION_COMMIT="${!EXTENSION_COMMIT_VAR}"; \ + git -C "extensions/${EXTENSION}" checkout "${EXTENSION_COMMIT}"; \ + git -C "extensions/${EXTENSION}" submodule update --init --recursive; \ + rm -rf "extensions/${EXTENSION}/.git*"; \ + find "extensions/${EXTENSION}" -name ".git*" -exec rm -rf {} +; \ + cp "/tmp/LocalSettings.d.template/${EXTENSION}.php" LocalSettings.d/; \ + done; \ + \ + git clone "https://github.com/ProfessionalWiki/WikibaseEdtf.git" "extensions/WikibaseEdtf"; \ + git -C "extensions/WikibaseEdtf" checkout "${WIKIBASEEDTF_COMMIT}"; \ + rm -rf "extensions/WikibaseEdtf/.git*"; \ + cp "/tmp/LocalSettings.d.template/WikibaseEdtf.php" LocalSettings.d/; \ + \ + git clone "https://github.com/ProfessionalWiki/WikibaseLocalMedia.git" "extensions/WikibaseLocalMedia"; \ + git -C "extensions/WikibaseLocalMedia" checkout "${WIKIBASELOCALMEDIA_COMMIT}"; \ + rm -rf "extensions/WikibaseLocalMedia/.git*"; \ + cp "/tmp/LocalSettings.d.template/WikibaseLocalMedia.php" LocalSettings.d/; \ + \ + rm -rf /var/www/html/vendor && \ + rm -rf /var/www/html/composer.lock && \ + composer install --no-dev -vv -n + +# ########################################################################### +FROM ${WIKIBASE_SUITE_WIKIBASE_IMAGE_URL}:${RELEASE_VERSION}-${WMDE_RELEASE_VERSION} +LABEL org.opencontainers.image.source="https://github.com/wmde/wikibase-release-pipeline" +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install \ + --yes --no-install-recommends lua5.1 && \ + rm -rf /var/lib/apt/lists/* &&\ + \ + rm -rf /var/www/html +COPY --from=composer /var/www/html /var/www/html +COPY extra-install/ /extra-install/ +COPY oauth.ini /templates/oauth.ini + +# MW_WG_UPLOAD_DIRECTORY set in original wikibase image +RUN chown www-data ${MW_WG_UPLOAD_DIRECTORY} -R diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/Babel.php b/build/WikibaseBundle/LocalSettings.d.template/Babel.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/Babel.php rename to build/WikibaseBundle/LocalSettings.d.template/Babel.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/CirrusSearch.php b/build/WikibaseBundle/LocalSettings.d.template/CirrusSearch.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/CirrusSearch.php rename to build/WikibaseBundle/LocalSettings.d.template/CirrusSearch.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/ConfirmEdit.php b/build/WikibaseBundle/LocalSettings.d.template/ConfirmEdit.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/ConfirmEdit.php rename to build/WikibaseBundle/LocalSettings.d.template/ConfirmEdit.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/Elastica.php b/build/WikibaseBundle/LocalSettings.d.template/Elastica.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/Elastica.php rename to build/WikibaseBundle/LocalSettings.d.template/Elastica.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/EntitySchema.php b/build/WikibaseBundle/LocalSettings.d.template/EntitySchema.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/EntitySchema.php rename to build/WikibaseBundle/LocalSettings.d.template/EntitySchema.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/Nuke.php b/build/WikibaseBundle/LocalSettings.d.template/Nuke.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/Nuke.php rename to build/WikibaseBundle/LocalSettings.d.template/Nuke.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/OAuth.php b/build/WikibaseBundle/LocalSettings.d.template/OAuth.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/OAuth.php rename to build/WikibaseBundle/LocalSettings.d.template/OAuth.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/Scribunto.php b/build/WikibaseBundle/LocalSettings.d.template/Scribunto.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/Scribunto.php rename to build/WikibaseBundle/LocalSettings.d.template/Scribunto.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/SyntaxHighlight_GeSHi.php b/build/WikibaseBundle/LocalSettings.d.template/SyntaxHighlight_GeSHi.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/SyntaxHighlight_GeSHi.php rename to build/WikibaseBundle/LocalSettings.d.template/SyntaxHighlight_GeSHi.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/UniversalLanguageSelector.php b/build/WikibaseBundle/LocalSettings.d.template/UniversalLanguageSelector.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/UniversalLanguageSelector.php rename to build/WikibaseBundle/LocalSettings.d.template/UniversalLanguageSelector.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/VisualEditor.php b/build/WikibaseBundle/LocalSettings.d.template/VisualEditor.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/VisualEditor.php rename to build/WikibaseBundle/LocalSettings.d.template/VisualEditor.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseCirrusSearch.php b/build/WikibaseBundle/LocalSettings.d.template/WikibaseCirrusSearch.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseCirrusSearch.php rename to build/WikibaseBundle/LocalSettings.d.template/WikibaseCirrusSearch.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseEdtf.php b/build/WikibaseBundle/LocalSettings.d.template/WikibaseEdtf.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseEdtf.php rename to build/WikibaseBundle/LocalSettings.d.template/WikibaseEdtf.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseLocalMedia.php b/build/WikibaseBundle/LocalSettings.d.template/WikibaseLocalMedia.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseLocalMedia.php rename to build/WikibaseBundle/LocalSettings.d.template/WikibaseLocalMedia.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseManifest.php b/build/WikibaseBundle/LocalSettings.d.template/WikibaseManifest.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/WikibaseManifest.php rename to build/WikibaseBundle/LocalSettings.d.template/WikibaseManifest.php diff --git a/Docker/build/WikibaseBundle/LocalSettings.d.template/cldr.php b/build/WikibaseBundle/LocalSettings.d.template/cldr.php similarity index 100% rename from Docker/build/WikibaseBundle/LocalSettings.d.template/cldr.php rename to build/WikibaseBundle/LocalSettings.d.template/cldr.php diff --git a/Docker/build/WikibaseBundle/README.md b/build/WikibaseBundle/README.md similarity index 100% rename from Docker/build/WikibaseBundle/README.md rename to build/WikibaseBundle/README.md diff --git a/Docker/build/WikibaseBundle/extra-install/ElasticSearch.sh b/build/WikibaseBundle/extra-install/ElasticSearch.sh similarity index 100% rename from Docker/build/WikibaseBundle/extra-install/ElasticSearch.sh rename to build/WikibaseBundle/extra-install/ElasticSearch.sh diff --git a/Docker/build/WikibaseBundle/extra-install/QuickStatements.sh b/build/WikibaseBundle/extra-install/QuickStatements.sh similarity index 100% rename from Docker/build/WikibaseBundle/extra-install/QuickStatements.sh rename to build/WikibaseBundle/extra-install/QuickStatements.sh diff --git a/build/WikibaseBundle/oauth.ini b/build/WikibaseBundle/oauth.ini new file mode 100644 index 000000000..557d34adb --- /dev/null +++ b/build/WikibaseBundle/oauth.ini @@ -0,0 +1,5 @@ +; HTTP User-Agent header +agent = 'Wikibase Docker QuickStatements' +; assigned by Special:OAuthConsumerRegistration (request modelled after https://www.wikidata.org/wiki/Special:OAuthListConsumers/view/77b4ae5506dd7dbb0bb07f80e3ae3ca9) +consumerKey = '${OAUTH_CONSUMER_KEY}' +consumerSecret = '${OAUTH_CONSUMER_SECRET}' diff --git a/build/build_elasticsearch_docker.sh b/build/build_elasticsearch_docker.sh deleted file mode 100644 index 5559b9cbb..000000000 --- a/build/build_elasticsearch_docker.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -e - -echo "Building elasticseach $ELASTICSEARCH_VERSION" -docker build \ - --build-arg=ELASTICSEARCH_VERSION="$ELASTICSEARCH_VERSION" \ - --build-arg=ELASTICSEARCH_PLUGIN_EXTRA_VERSION="$ELASTICSEARCH_PLUGIN_EXTRA_VERSION" \ - Docker/build/Elasticsearch/ -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE" > "$(pwd)"/artifacts/elasticsearch.docker.tar.gz diff --git a/build/build_extension.sh b/build/build_extension.sh deleted file mode 100755 index 3bb511e75..000000000 --- a/build/build_extension.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -ROOT="$(pwd)" - -EXTENSION=$1 -BRANCH_NAME=$2 -FINAL_EXTENSION_PATH=$3 - -TEMP_GIT_DIR="$(mktemp -d)" -TEMP_TAR_DIR="$(pwd)"/artifacts -EXT_PATH="$TEMP_GIT_DIR/$EXTENSION" -EXTENSION_COMMIT_HASH_VARIABLE="${EXTENSION^^}_COMMIT_HASH" - -mkdir -p "$TEMP_TAR_DIR" - -COMMIT_HASH=${!EXTENSION_COMMIT_HASH_VARIABLE} -UPDATE_SUBMODULE=1 bash "$ROOT"/build/clone_repo.sh \ - "$COMMIT_HASH" \ - "$ROOT/git_cache/$EXTENSION.git" \ - "$EXTENSION" \ - "$EXT_PATH" \ - "${BRANCH_NAME}" - -bash "$ROOT"/build/clean_repo.sh "$EXT_PATH" - -cp -r "$EXT_PATH" "$FINAL_EXTENSION_PATH" diff --git a/build/build_quickstatements.sh b/build/build_quickstatements.sh deleted file mode 100755 index ac4d87458..000000000 --- a/build/build_quickstatements.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash -set -e - -ROOT="$(pwd)" -TARBALL=quickstatements.tar.gz -TARBALL_PATH="$ROOT/artifacts/$TARBALL" - -# clone and clean quickstatements -TEMP_GIT_DIR=$(mktemp -d) -QS_GIT_DIR="$TEMP_GIT_DIR/quickstatements" -UPDATE_SUBMODULE=1 bash "$ROOT"/build/clone_repo.sh "$QUICKSTATEMENTS_COMMIT_HASH" "$ROOT/git_cache/services/quickstatements.git" quickstatements "$QS_GIT_DIR" master -bash "$ROOT"/build/clean_repo.sh "$QS_GIT_DIR" - -# clone and clean magnustools -MAGNUSTOOLS_GIT_DIR="$TEMP_GIT_DIR/magnustools" -UPDATE_SUBMODULE=0 bash "$ROOT"/build/clone_repo.sh "$MAGNUSTOOLS_COMMIT_HASH" "$ROOT/git_cache/services/magnustools.git" magnustools "$MAGNUSTOOLS_GIT_DIR" master -bash "$ROOT"/build/clean_repo.sh "$MAGNUSTOOLS_GIT_DIR" - -cd "$TEMP_GIT_DIR" -tar -C "$TEMP_GIT_DIR" -cvf - magnustools quickstatements | gzip -"$GZIP_COMPRESSION_RATE" > "$TARBALL_PATH" -cd - - -if [ -n "$GITHUB_ENV" ]; then - echo "TARBALL_PATH=$TARBALL_PATH" >> "$GITHUB_ENV" -else - export TARBALL_PATH -fi diff --git a/build/build_quickstatements_docker.sh b/build/build_quickstatements_docker.sh deleted file mode 100755 index f9ea7e63e..000000000 --- a/build/build_quickstatements_docker.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -ex - -mkdir -p Docker/build/QuickStatements/artifacts -cp "$TARBALL_PATH" Docker/build/QuickStatements/artifacts/ - -docker build \ - --build-arg COMPOSER_IMAGE_NAME="$COMPOSER_IMAGE_NAME" \ - --build-arg COMPOSER_IMAGE_VERSION="$COMPOSER_IMAGE_VERSION" \ - --no-cache Docker/build/QuickStatements/ -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE" > artifacts/"$1".docker.tar.gz diff --git a/build/build_wdqs.sh b/build/build_wdqs.sh deleted file mode 100755 index 092820ab9..000000000 --- a/build/build_wdqs.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# Build script for WDQS -# Downloads the service-dist.tar file from archiva and caches it -set -e - -TARBALL="service-$WDQS_VERSION-dist.tar.gz" -TARBALL_SIGNATURE="$TARBALL".md5 - -MD5_SIGNATURE="$(wget -qO- "https://archiva.wikimedia.org/repository/releases/org/wikidata/query/rdf/service/$WDQS_VERSION/$TARBALL_SIGNATURE")" -CACHE_PATH="cache/wdqs/$WDQS_VERSION" - -if [ ! -f "$CACHE_PATH/$TARBALL" ]; then - mkdir -p "$CACHE_PATH" - wget "https://archiva.wikimedia.org/repository/releases/org/wikidata/query/rdf/service/$WDQS_VERSION/$TARBALL" \ - -O "$CACHE_PATH/$TARBALL" -fi - -echo "$MD5_SIGNATURE $CACHE_PATH/$TARBALL" | md5sum -c - -TARBALL_PATH="$CACHE_PATH/$TARBALL" - -if [ -n "$GITHUB_ENV" ]; then - echo "TARBALL_PATH=$TARBALL_PATH" >> "$GITHUB_ENV" -else - export TARBALL_PATH -fi diff --git a/build/build_wdqs_docker.sh b/build/build_wdqs_docker.sh deleted file mode 100755 index 62c600802..000000000 --- a/build/build_wdqs_docker.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -e - -SERVICE_DIST_TAR="$TARBALL_PATH" - -cp "$SERVICE_DIST_TAR" Docker/build/WDQS/ -cp Docker/build/wait-for-it.sh Docker/build/WDQS/ - -docker build --pull --build-arg tarball="$(basename "$SERVICE_DIST_TAR")" Docker/build/WDQS/ -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE" > artifacts/"$1".docker.tar.gz diff --git a/build/build_wdqs_frontend.sh b/build/build_wdqs_frontend.sh deleted file mode 100755 index d56d872a6..000000000 --- a/build/build_wdqs_frontend.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -set -ex - -ROOT="$(pwd)" -WDQS_FRONTEND_GIT_DIR="$(mktemp -d)" -TARBALL_PATH="$ROOT"/artifacts/wdqs-frontend.tar.gz - -UPDATE_SUBMODULE=1 bash "$ROOT"/build/clone_repo.sh \ - "$WDQS_FRONTEND_COMMIT_HASH" \ - "$ROOT/git_cache/services/wikidata-query-gui.git" \ - WDQS_FRONTEND \ - "$WDQS_FRONTEND_GIT_DIR" \ - master - -bash "$ROOT"/build/clean_repo.sh "$WDQS_FRONTEND_GIT_DIR" - -cd "$WDQS_FRONTEND_GIT_DIR" -tar -cvf - -- * | gzip -"$GZIP_COMPRESSION_RATE" > "$TARBALL_PATH" -cd "$ROOT" - -if [ -n "$GITHUB_ENV" ]; then - echo "TARBALL_PATH=$TARBALL_PATH" >> "$GITHUB_ENV" -else - export TARBALL_PATH -fi diff --git a/build/build_wdqs_frontend_docker.sh b/build/build_wdqs_frontend_docker.sh deleted file mode 100755 index c0d3f792e..000000000 --- a/build/build_wdqs_frontend_docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -set -e - -cp -r "$TARBALL_PATH" Docker/build/WDQS-frontend -docker build --pull --build-arg tarball="$(basename "$TARBALL_PATH")" Docker/build/WDQS-frontend/ -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE"f > "$(pwd)"/artifacts/"$1".docker.tar.gz diff --git a/build/build_wdqs_proxy_docker.sh b/build/build_wdqs_proxy_docker.sh deleted file mode 100755 index 03b58301c..000000000 --- a/build/build_wdqs_proxy_docker.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -ex - -docker build \ - Docker/build/WDQS-proxy/ -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE" > artifacts/"$1".docker.tar.gz diff --git a/build/build_wikibase.sh b/build/build_wikibase.sh deleted file mode 100755 index 2ae8e3347..000000000 --- a/build/build_wikibase.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -ex - -ROOT="$(pwd)" - -ARTIFACT_PATH="$(pwd)/artifacts" -WIKIBASE_PATH="/tmp/Wikibase" -WIKIBASE_PATH_HOST="${HOST_TMP}/Wikibase" - -UPDATE_SUBMODULE=1 bash "$ROOT"/build/clone_repo.sh \ - "$WIKIBASE_COMMIT_HASH" \ - "$ROOT/git_cache/Wikibase.git" \ - WIKIBASE \ - "$WIKIBASE_PATH" \ - "${WIKIBASE_BRANCH_NAME}" - -bash "$ROOT"/build/clean_repo.sh "$WIKIBASE_PATH" - -# remove travis build file -rm "$WIKIBASE_PATH"/.travis.yml -vf - -COMPOSER_VENDOR="$WIKIBASE_PATH/vendor/" - -mkdir "$COMPOSER_VENDOR" -chmod 777 "$COMPOSER_VENDOR" - -# TODO remove the below hack: -# composer config --no-plugins allow-plugins.composer/installers false -# Note: we need to provide the host path for setting up the volume, as we -# are currently already in the context of a container, we cannot just use -# the current path from the current context -docker run --rm \ - --volume "$WIKIBASE_PATH_HOST":/tmp/Wikibase \ - -u "$(id -u)":"$(id -g)" \ - "$COMPOSER_IMAGE_NAME:$COMPOSER_IMAGE_VERSION" \ - config --no-plugins allow-plugins.composer/installers false -d "/tmp/Wikibase" -docker run --rm \ - --volume "$WIKIBASE_PATH_HOST":/tmp/Wikibase \ - -u "$(id -u)":"$(id -g)" \ - "$COMPOSER_IMAGE_NAME:$COMPOSER_IMAGE_VERSION" \ - install --no-dev --ignore-platform-reqs -vv -d "/tmp/Wikibase" -chmod 755 "$COMPOSER_VENDOR" - -tar -C "$(dirname "$WIKIBASE_PATH")" -cf - Wikibase | gzip -"$GZIP_COMPRESSION_RATE" > "$ARTIFACT_PATH"/wikibase.tar.gz - -TARBALL_PATH="$ARTIFACT_PATH/wikibase.tar.gz" - -if [ -n "$GITHUB_ENV" ]; then - echo "TARBALL_PATH=$TARBALL_PATH" >> "$GITHUB_ENV" -else - export TARBALL_PATH -fi diff --git a/build/build_wikibase_bundle_docker.sh b/build/build_wikibase_bundle_docker.sh deleted file mode 100755 index aa9b4a268..000000000 --- a/build/build_wikibase_bundle_docker.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090 -set -e - -BUILD_TMP="/tmp/build-wikibase-bundle-docker" - -mkdir -p "${BUILD_TMP}" -cp -r Docker/build/WikibaseBundle/* "${BUILD_TMP}" - -BUILT_EXTENSIONS_PATH="${BUILD_TMP}/artifacts/extensions" - -mkdir -p "$BUILT_EXTENSIONS_PATH" - -docker load -i "artifacts/wikibase.docker.tar.gz" - -## copy oauth template to build artifacts -cp Docker/build/QuickStatements/oauth.ini "${BUILD_TMP}/artifacts/" - -## Create LocalSettings dir in build folder -mkdir -p "${BUILD_TMP}/LocalSettings.d/" - -## If BUNDLE_WMF_EXTENSIONS not defined fallback to default -if [ -z "$BUNDLE_WMF_EXTENSIONS" ]; then - export BUNDLE_WMF_EXTENSIONS="$DEFAULT_BUNDLE_WMF_EXTENSIONS" -fi - -## If the $GERRIT_EXTENSION_BRANCH_NAME override is set, use that branch -if [ -z "$GERRIT_EXTENSION_BRANCH_NAME" ]; then - GERRIT_EXTENSION_BRANCH_NAME="$WIKIBASE_BRANCH_NAME" -fi - -## Extensions from gerrit -IFS=',' read -ra EXTENSIONS <<< "$BUNDLE_WMF_EXTENSIONS" -for EXTENSION in "${EXTENSIONS[@]}"; do - bash build/build_extension.sh "$EXTENSION" "${GERRIT_EXTENSION_BRANCH_NAME}" "$BUILT_EXTENSIONS_PATH" - ## Copy the configuration files to build directory - cp "Docker/build/WikibaseBundle/LocalSettings.d.template/${EXTENSION}.php" "${BUILD_TMP}/LocalSettings.d/" -done - -## If BUNDLE_EXT_EXTENSIONS not defined fallback to default -if [ -z "$BUNDLE_EXT_EXTENSIONS" ]; then - export BUNDLE_EXT_EXTENSIONS="$DEFAULT_BUNDLE_EXT_EXTENSIONS" -fi - -## External extensions are built from individual scripts -## Example: build/external_extensions/.sh -IFS=',' read -ra EXT_EXTENSIONS <<< "$BUNDLE_EXT_EXTENSIONS" -for EXT_EXTENSION in "${EXT_EXTENSIONS[@]}"; do - - ## build external extension - . "build/external_extension/${EXT_EXTENSION}.sh" - ## Copy the configuration files to build directory - cp "Docker/build/WikibaseBundle/LocalSettings.d.template/${EXT_EXTENSION}.php" "$BUILD_TMP/LocalSettings.d/" -done - -docker build --no-cache \ - --build-arg WIKIBASE_IMAGE_NAME="$WIKIBASE_IMAGE_NAME" \ - --build-arg COMPOSER_IMAGE_NAME="$COMPOSER_IMAGE_NAME" \ - --build-arg COMPOSER_IMAGE_VERSION="$COMPOSER_IMAGE_VERSION" \ - "${BUILD_TMP}" -t "$WIKIBASE_BUNDLE_IMAGE_NAME" - -docker save "$WIKIBASE_BUNDLE_IMAGE_NAME" | gzip -"$GZIP_COMPRESSION_RATE" > artifacts/"$WIKIBASE_BUNDLE_IMAGE_NAME".docker.tar.gz diff --git a/build/build_wikibase_docker.sh b/build/build_wikibase_docker.sh deleted file mode 100755 index 178b1bac4..000000000 --- a/build/build_wikibase_docker.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1091 -set -e - -BUILD_TMP="/tmp/build-wikibase-docker" - -mkdir -p "${BUILD_TMP}" -cp -r Docker/build/Wikibase/* "${BUILD_TMP}" - -mkdir -p "${BUILD_TMP}/artifacts/extensions" - -MEDIAWIKI_IMAGE_VERSION="$MEDIAWIKI_VERSION" - -if [ -f "$TARBALL_PATH" ]; then - cp "$TARBALL_PATH" "${BUILD_TMP}/artifacts/" -fi - -cp Docker/build/wait-for-it.sh "${BUILD_TMP}/artifacts/" -set -o allexport; source Docker/build/Wikibase/default.env; set +o allexport - -docker build \ - --build-arg MEDIAWIKI_IMAGE_NAME="$MEDIAWIKI_IMAGE_NAME" \ - --build-arg MEDIAWIKI_IMAGE_VERSION="$MEDIAWIKI_IMAGE_VERSION" \ - --build-arg COMPOSER_IMAGE_NAME="$COMPOSER_IMAGE_NAME" \ - --build-arg COMPOSER_IMAGE_VERSION="$COMPOSER_IMAGE_VERSION" \ - --build-arg MEDIAWIKI_SETTINGS_TEMPLATE_FILE="$MEDIAWIKI_SETTINGS_TEMPLATE_FILE" \ - \ - --build-arg MW_SITE_NAME="$MW_SITE_NAME" \ - --build-arg MW_SITE_LANG="$MW_SITE_LANG" \ - --build-arg MW_WG_JOB_RUN_RATE="$MW_WG_JOB_RUN_RATE" \ - --build-arg MW_WG_ENABLE_UPLOADS="$MW_WG_ENABLE_UPLOADS" \ - --build-arg MW_WG_UPLOAD_DIRECTORY="$MW_WG_UPLOAD_DIRECTORY" \ - --build-arg WIKIBASE_PINGBACK="$WIKIBASE_PINGBACK" \ - \ - "${BUILD_TMP}" -t "$1" - -docker save "$1" | gzip -"$GZIP_COMPRESSION_RATE" > artifacts/"$1".docker.tar.gz diff --git a/build/clean_repo.sh b/build/clean_repo.sh deleted file mode 100755 index 5b6f5c0d1..000000000 --- a/build/clean_repo.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# Cleans repository of undesired files -set -ex - -REPO_DIR=$1 - -# remove git things from release package -rm "$REPO_DIR"/.git* -rf diff --git a/build/clone_repo.sh b/build/clone_repo.sh deleted file mode 100755 index 92196cfe6..000000000 --- a/build/clone_repo.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env bash -# Clones a repository and writes the metadata file -set -e - -COMMIT_HASH=$1 -GIT_REPO=$2 -NAME=$3 -TEMP_GIT_DIR=$4 -BRANCH=$5 - -ROOT="$(pwd)" -BUILD_METADATA_ENV_FILE="$ROOT/artifacts/build_metadata_${NAME,,}.env" -BUILD_METADATA_ENV_NAME="${NAME^^}_COMMIT_HASH" - -git clone --single-branch --branch "$BRANCH" "$GIT_REPO" "$TEMP_GIT_DIR" - -cd "$TEMP_GIT_DIR" - -# either use HEAD on master, or tied to a specific commit -if [ -n "$COMMIT_HASH" ]; then - git reset --hard "$COMMIT_HASH" - -else - echo "COMMIT_HASH not set using HEAD of $BRANCH." - COMMIT_HASH="$(git rev-parse HEAD)" -fi - -# log the selected commit -echo -echo "Checking out $NAME at" -git show "$COMMIT_HASH" --quiet -echo - -if [ "$UPDATE_SUBMODULE" = 1 ] ; then - git submodule update --init --recursive -fi - -cd - - -bash "$ROOT"/build/write_git_metadata.sh "$TEMP_GIT_DIR" "$BUILD_METADATA_ENV_FILE" "$BUILD_METADATA_ENV_NAME" "$COMMIT_HASH" diff --git a/build/external_extension/WikibaseEdtf.sh b/build/external_extension/WikibaseEdtf.sh deleted file mode 100644 index 75dcb7856..000000000 --- a/build/external_extension/WikibaseEdtf.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -e - -## WikibaseEDTF -UPDATE_SUBMODULE=0 bash build/clone_repo.sh \ - "$WIKIBASEEDTF_COMMIT_HASH" \ - "git_cache/WikibaseEdtf.git" \ - WIKIBASEEDTF \ - "$BUILT_EXTENSIONS_PATH/WikibaseEdtf" \ - master - -bash build/clean_repo.sh "$BUILT_EXTENSIONS_PATH/WikibaseEdtf" diff --git a/build/external_extension/WikibaseLocalMedia.sh b/build/external_extension/WikibaseLocalMedia.sh deleted file mode 100644 index c38671dc7..000000000 --- a/build/external_extension/WikibaseLocalMedia.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -e - -## WikibaseLocalMedia -## NOTE: WikibaseLocalMedia does currently not work in a client only setup. -UPDATE_SUBMODULE=0 bash build/clone_repo.sh \ - "$WIKIBASELOCALMEDIA_COMMIT_HASH" \ - "git_cache/WikibaseLocalMedia.git" \ - WIKIBASELOCALMEDIA \ - "$BUILT_EXTENSIONS_PATH/WikibaseLocalMedia" \ - master - -bash build/clean_repo.sh "$BUILT_EXTENSIONS_PATH/WikibaseLocalMedia" diff --git a/build/requirements/build_version_requirements.py b/build/requirements/build_version_requirements.py deleted file mode 100644 index 056085bca..000000000 --- a/build/requirements/build_version_requirements.py +++ /dev/null @@ -1,31 +0,0 @@ -# TODO: https://phabricator.wikimedia.org/T347084 -import yaml -import os - -def getVar( varName ): - if varName in os.environ: - name = components[varName] - return name + " " + os.environ[varName] - else: - return 'undefined' - -components = { - "RELEASE_MAJOR_VERSION": "MediaWiki", - "ELASTICSEARCH_VERSION": "Elasticsearch", - "WDQS_VERSION": "WDQS", -} - -databases = [] - -# read databases -with open('config.yml') as f: - data = yaml.load(f, Loader=yaml.FullLoader) - databases = data['jobs']['test']['strategy']['matrix']['databaseImageName'] - -f = open("artifacts/built_versions.log", "w+") - -for key in components: - f.write(getVar(key) + '\n') - -f.write('Databases: ' + ', '.join(databases)) -f.close() diff --git a/build/write_git_metadata.sh b/build/write_git_metadata.sh deleted file mode 100755 index 1d4ea9446..000000000 --- a/build/write_git_metadata.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -set -e - -REPO_PATH=$1 -BUILD_METADATA_ENV_FILE=$2 -VARIABLE_NAME="METADATA_$3" - -# if passed as 4th argument this is the hash -if [ -z "$4" ]; then - REPO_GIT_HASH="$(git -C "$REPO_PATH" rev-parse HEAD)" -else - REPO_GIT_HASH=$4 -fi - -if [ -f "$BUILD_METADATA_ENV_FILE" ]; then - sed -i "/$VARIABLE_NAME/d" "$BUILD_METADATA_ENV_FILE" -fi - -echo "$VARIABLE_NAME=$REPO_GIT_HASH" >> "$BUILD_METADATA_ENV_FILE" diff --git a/cache/.keep b/cache/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/docs/diagrams/README.md b/docs/diagrams/README.md index c1062cf97..c189b4a58 100644 --- a/docs/diagrams/README.md +++ b/docs/diagrams/README.md @@ -17,7 +17,3 @@ If you get a Chrome error `Running as root without --no-sandbox is not supported `docker run -it --rm -v "$PWD:/app" diagrams node_modules/.bin/mmdc -p puppeteer-config.json -i main.mmd -o output.svg` -## Building diagrams for documentation - -There is a `build.sh` script to build all documentation diagrams in one go by running. This can be run either inside or outside the docker container. - diff --git a/docs/topics/pipeline.md b/docs/topics/pipeline.md index de106c8e6..edf38986f 100644 --- a/docs/topics/pipeline.md +++ b/docs/topics/pipeline.md @@ -40,26 +40,37 @@ After triggering the pipeline a set of jobs will start running, which after a su ## Running the pipeline locally -To execute the entire build pipeline you can use the `all` make target. +To execute the entire build pipeline, just run `build.sh`. ```sh -./build.sh all +./build.sh ``` -To build single artifacts locally you can issue the following command. +To build single artifacts locally, you can issue the following command. ```sh ./build.sh wikibase ``` -The output ends up in the `artifacts/` folder and will be built as described by the `variables.env` environment file. +When building locally the artifacts are only stored in the docker daemons image storage. If you want to create files as well, use the `--save-image` option with `build.sh`. +```sh +# build and store results in artifacts folder +$ ./build.sh --save-image +``` + +If you want to also extract standalone tarballs, use the following command. This is basically also what the CI calls. + +```sh +$ ./build.sh --save-image --extract-tarball +``` -See the [Makefile](../../Makefile) for more build options. +To rebuild without using Dockers cache, add the `--no-cache` option. Note that this will extend build times as all components need to be downloaded again (except Docker base images). -## Local artifacts +```sh +$ ./build.sh --no-cache +``` -When building locally the artifacts will be output into the `artifacts/` folder. They are not zip'd as on Github. ### Clean local artifacts @@ -69,12 +80,6 @@ To remove any locally produced artifacts you can run the following commands. make clean ``` -To clean the git_cache used for building - -```sh -make clean-cache -``` - ### Downloaded artifacts Any downloaded artifacts from workflow runs on Github actions will be saved as a folder under `artifacts/` and are exempt from any cleaning commands. diff --git a/docs/topics/security-releases.md b/docs/topics/security-releases.md index 0b01fbf2c..25a3b2c22 100644 --- a/docs/topics/security-releases.md +++ b/docs/topics/security-releases.md @@ -14,10 +14,9 @@ You can see all new commits since the commit being used using a comparison URL ( You can then use a new commit hash that only includes minor fixes including security updates to packages and code. -- `WDQS_FRONTEND_COMMIT_HASH`: https://github.com/wikimedia/wikidata-query-gui/compare/3be93201ca2efad44f36430b8cf3a2c94cb22ebf...master -- `QUICKSTATEMENTS_COMMIT_HASH`: https://github.com/magnusmanske/quickstatements/compare/757d238cf6d306f9daf8276f620415cf09f4afe6...master -- `MAGNUSTOOLS_COMMIT_HASH`: https://bitbucket.org/magnusmanske/magnustools/branches/compare/master..7db2f401390df471d9650010ce059e4308d0cc9e This repository includes lots of code that is not being used withing quicksttaements. Changes to this code could be ignored. -- `WIKIBASELOCALMEDIA_COMMIT_HASH` +- `WDQSQUERYGUI_COMMIT`: https://github.com/wikimedia/wikidata-query-gui/compare/3be93201ca2efad44f36430b8cf3a2c94cb22ebf...master +- `QUICKSTATEMENTS_COMMIT`: https://github.com/magnusmanske/quickstatements/compare/757d238cf6d306f9daf8276f620415cf09f4afe6...master +- `MAGNUSTOOLS_COMMIT`: https://bitbucket.org/magnusmanske/magnustools/branches/compare/master..7db2f401390df471d9650010ce059e4308d0cc9e This repository includes lots of code that is not being used withing quicksttaements. Changes to this code could be ignored. **No specific security releases** diff --git a/docs/topics/upgrading.md b/docs/topics/upgrading.md index f7bc2fed7..42ed68c16 100644 --- a/docs/topics/upgrading.md +++ b/docs/topics/upgrading.md @@ -11,7 +11,7 @@ In all of our images we rely on a database to persist data. Normally these are s ```yml mysql: - image: "${DEFAULT_DATABASE_IMAGE_NAME}" + image: "${MARIADB_IMAGE_URL}" restart: unless-stopped volumes: - mediawiki-mysql-data:/var/lib/mysql diff --git a/flake.nix b/flake.nix index 621c334c5..003ea9ef7 100644 --- a/flake.nix +++ b/flake.nix @@ -22,8 +22,16 @@ echo echo "Build Commands:" - echo "$ ./build.sh all" + echo "# Build all wikibase suite components docker images" + echo "$ ./build.sh" + echo "# Build only the mediawiki/wikibase containers" echo "$ ./build.sh wikibase" + echo "# Build only the query service container and save the docker image to a tarball" + echo "$ ./build.sh --save-image wdqs" + echo "# Build the wdqs-frontend container and extract a standalone tarball from the webroot" + echo "$ ./build.sh --extract-tarball wdqs-frontend" + echo "# Build the wdqs container without using Dockers cache" + echo "$ ./build.sh --no-cache wdqs" echo echo "Test Commands:" echo "$ make test" diff --git a/git_cache/.keep b/git_cache/.keep deleted file mode 100644 index e69de29bb..000000000 diff --git a/publish/docker-compose.yml b/publish/docker-compose.yml index 0c6f437ca..11c3b0c2c 100644 --- a/publish/docker-compose.yml +++ b/publish/docker-compose.yml @@ -3,7 +3,7 @@ version: '3' services: download_artifacts: build: - context: ../Docker/publish/download_artifacts + context: download_artifacts dockerfile: Dockerfile environment: - GITHUB_TOKEN @@ -12,7 +12,7 @@ services: - ../artifacts:/extractedArtifacts upload_dockerhub: build: - context: ../Docker/publish/upload_dockerhub + context: upload_dockerhub dockerfile: Dockerfile privileged: true environment: @@ -24,13 +24,13 @@ services: - QUICKSTATEMENTS_DOCKER_PATH=/extractedArtifacts/DockerImages/quickstatements.docker.tar.gz - WDQS_PROXY_DOCKER_PATH=/extractedArtifacts/DockerImages/wdqs-proxy.docker.tar.gz - - WDQS_IMAGE_NAME - - WDQS_FRONTEND_IMAGE_NAME - - WDQS_PROXY_IMAGE_NAME - - QUICKSTATEMENTS_IMAGE_NAME - - WIKIBASE_IMAGE_NAME - - WIKIBASE_BUNDLE_IMAGE_NAME - - ELASTICSEARCH_IMAGE_NAME + - WIKIBASE_SUITE_WDQS_IMAGE_URL + - WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL + - WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL + - WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL + - WIKIBASE_SUITE_WIKIBASE_IMAGE_URL + - WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL + - WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL - WDQS_VERSION - RELEASE_VERSION @@ -46,7 +46,7 @@ services: - ../artifacts/${WORKFLOW_RUN_NUMBER}:/extractedArtifacts upload_tar: build: - context: ../Docker/publish/upload_tar + context: /upload_tar dockerfile: Dockerfile privileged: true environment: diff --git a/Docker/publish/download_artifacts/Dockerfile b/publish/download_artifacts/Dockerfile similarity index 100% rename from Docker/publish/download_artifacts/Dockerfile rename to publish/download_artifacts/Dockerfile diff --git a/Docker/publish/download_artifacts/download_artifacts.py b/publish/download_artifacts/download_artifacts.py similarity index 100% rename from Docker/publish/download_artifacts/download_artifacts.py rename to publish/download_artifacts/download_artifacts.py diff --git a/Docker/publish/upload_dockerhub/Dockerfile b/publish/upload_dockerhub/Dockerfile similarity index 100% rename from Docker/publish/upload_dockerhub/Dockerfile rename to publish/upload_dockerhub/Dockerfile diff --git a/Docker/publish/upload_dockerhub/publish.sh b/publish/upload_dockerhub/publish.sh similarity index 68% rename from Docker/publish/upload_dockerhub/publish.sh rename to publish/upload_dockerhub/publish.sh index 896762403..449e62cff 100644 --- a/Docker/publish/upload_dockerhub/publish.sh +++ b/publish/upload_dockerhub/publish.sh @@ -9,13 +9,13 @@ if [ -z "$WDQS_DOCKER_PATH" ] || \ [ -z "$QUICKSTATEMENTS_DOCKER_PATH" ] || \ [ -z "$WDQS_PROXY_DOCKER_PATH" ] || \ \ -[ -z "$WDQS_FRONTEND_IMAGE_NAME" ] || \ -[ -z "$WDQS_IMAGE_NAME" ] || \ -[ -z "$WDQS_PROXY_IMAGE_NAME" ] || \ -[ -z "$WIKIBASE_IMAGE_NAME" ] || \ -[ -z "$WIKIBASE_BUNDLE_IMAGE_NAME" ] || \ -[ -z "$ELASTICSEARCH_IMAGE_NAME" ] || \ -[ -z "$QUICKSTATEMENTS_IMAGE_NAME" ] || \ +[ -z "$WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_WDQS_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL" ] || \ +[ -z "$WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL" ] || \ \ [ -z "$RELEASE_VERSION" ] || \ [ -z "$WMDE_RELEASE_VERSION" ] || \ @@ -29,13 +29,12 @@ if [ -z "$WDQS_DOCKER_PATH" ] || \ exit 1; fi -function tag_and_push { +function push { IMAGE_NAME=$1 IMAGE_VERSION=$2 # example: wikibase/wikibase:1.35 IMAGE_TAG="$DOCKER_HUB_REPOSITORY_NAME/$IMAGE_NAME:$IMAGE_VERSION" - docker tag "$IMAGE_NAME:latest" "$IMAGE_TAG" if [ -z "$DRY_RUN" ]; then docker push "$IMAGE_TAG" @@ -66,25 +65,25 @@ docker load -i "$QUICKSTATEMENTS_DOCKER_PATH" docker load -i "$WDQS_PROXY_DOCKER_PATH" # Tag Queryservice Proxy with version -tag_and_push "$WDQS_PROXY_IMAGE_NAME" "$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL" "$WMDE_RELEASE_VERSION" # Tag WDQS-frontend with version -tag_and_push "$WDQS_FRONTEND_IMAGE_NAME" "$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL" "$WMDE_RELEASE_VERSION" # Tag WDQS with version -tag_and_push "$WDQS_IMAGE_NAME" "$WDQS_VERSION-$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_WDQS_IMAGE_URL" "$WDQS_VERSION-$WMDE_RELEASE_VERSION" # Tag Wikibase with version -tag_and_push "$WIKIBASE_IMAGE_NAME" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" # Tag Wikibase-bundle with version -tag_and_push "$WIKIBASE_BUNDLE_IMAGE_NAME" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL" "$RELEASE_VERSION-$WMDE_RELEASE_VERSION" # Tag Elasticsearch with version -tag_and_push "$ELASTICSEARCH_IMAGE_NAME" "$ELASTICSEARCH_VERSION-$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL" "$ELASTICSEARCH_VERSION-$WMDE_RELEASE_VERSION" # Tag Quickstatements with version -tag_and_push "$QUICKSTATEMENTS_IMAGE_NAME" "$WMDE_RELEASE_VERSION" +push "$WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL" "$WMDE_RELEASE_VERSION" # logout and remove credentials docker logout diff --git a/Docker/publish/upload_tar/Dockerfile b/publish/upload_tar/Dockerfile similarity index 100% rename from Docker/publish/upload_tar/Dockerfile rename to publish/upload_tar/Dockerfile diff --git a/Docker/publish/upload_tar/publish.sh b/publish/upload_tar/publish.sh similarity index 100% rename from Docker/publish/upload_tar/publish.sh rename to publish/upload_tar/publish.sh diff --git a/test/scripts/test_suite.sh b/test/scripts/test_suite.sh index f37de0cfa..8954aefea 100755 --- a/test/scripts/test_suite.sh +++ b/test/scripts/test_suite.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -e +# set -x cd test @@ -17,16 +18,18 @@ docker compose run --rm test-runner -c "rm -rf \"$RESULTS_DIR\"" > /dev/null 2>& mkdir -p "$RESULTS_DIR" # Load image if it's not already loaded -load_image() { +load_image() { local image="$1" - + # Check if the image exists { if docker images -q "$image" 2>/dev/null | grep -q .; then echo "ℹ️ Image $image already loaded." else + local image_filename echo "🔄 Loading image: $image" - docker load -i "../artifacts/$image.docker.tar.gz" + image_filename="../artifacts/$(echo "$image" | cut -d'/' -f2).docker.tar.gz" + docker load -i "$image_filename" fi } >> "$TEST_LOG" 2>&1 } @@ -34,27 +37,27 @@ load_image() { echo -e "\n▶️ Setting-up \"$SUITE\" test suite" 2>&1 | tee -a "$TEST_LOG" if [ -z "$DATABASE_IMAGE_NAME" ]; then - export DATABASE_IMAGE_NAME="$DEFAULT_DATABASE_IMAGE_NAME" + export DATABASE_IMAGE_NAME="$MARIADB_IMAGE_URL" fi # select image based on prepended suite name if [[ $SUITE == base__* ]]; then - WIKIBASE_TEST_IMAGE_NAME="$WIKIBASE_IMAGE_NAME" + WIKIBASE_TEST_IMAGE_NAME="$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" else - WIKIBASE_TEST_IMAGE_NAME="$WIKIBASE_BUNDLE_IMAGE_NAME" + WIKIBASE_TEST_IMAGE_NAME="$WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL" fi export WIKIBASE_TEST_IMAGE_NAME default_images=( "$WIKIBASE_TEST_IMAGE_NAME" - "$WDQS_IMAGE_NAME" - "$WDQS_FRONTEND_IMAGE_NAME" - "$WDQS_PROXY_IMAGE_NAME" + "$WIKIBASE_SUITE_WDQS_IMAGE_URL" + "$WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL" + "$WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL" ) bundle_images=( - "$ELASTICSEARCH_IMAGE_NAME" - "$QUICKSTATEMENTS_IMAGE_NAME" + "$WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL" + "$WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL" ) for image in "${default_images[@]}"; do @@ -70,14 +73,6 @@ fi echo "ℹ️ $(docker --version)" 2>&1 | tee -a "$TEST_LOG" -# Does it do anything to be adding the ":latest" tag to these? -export WIKIBASE_TEST_IMAGE_NAME="$WIKIBASE_TEST_IMAGE_NAME:latest" -export QUERYSERVICE_IMAGE_NAME="$QUERYSERVICE_IMAGE_NAME:latest" -export QUERYSERVICE_UI_IMAGE_NAME="$QUERYSERVICE_UI_IMAGE_NAME:latest" -export WDQS_PROXY_IMAGE_NAME="$WDQS_PROXY_IMAGE_NAME:latest" -export QUICKSTATEMENTS_IMAGE_NAME="$QUICKSTATEMENTS_IMAGE_NAME:latest" -export ELASTICSEARCH_IMAGE_NAME="$ELASTICSEARCH_IMAGE_NAME:latest" - export DEFAULT_SUITE_CONFIG="--env-file default.env -f suites/docker-compose.yml" bash scripts/run_selenium.sh "$SUITE" diff --git a/test/scripts/test_upgrade.sh b/test/scripts/test_upgrade.sh index 183c90a0e..76596d2b8 100755 --- a/test/scripts/test_upgrade.sh +++ b/test/scripts/test_upgrade.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # shellcheck disable=SC1091,SC1090,SC2086 set -e +# set -x cd test @@ -90,8 +91,8 @@ TMP_DIR="$(mktemp -d)" TMP_LOCALSETTINGS="$TMP_DIR/LocalSettings.php" # Source the default env vars used for building and create a new LocalSettings file -set -o allexport; source ../Docker/build/Wikibase/default.env; set +o allexport -envsubst < "../Docker/build/Wikibase/$MEDIAWIKI_SETTINGS_TEMPLATE_FILE" > "$TMP_LOCALSETTINGS" +set -o allexport; source ../variables.env; set +o allexport +envsubst < "../build/Wikibase/LocalSettings.php.template" > "$TMP_LOCALSETTINGS" export TMP_LOCALSETTINGS # MODIFY OLD LocalSettings.php as part of upgrading @@ -123,21 +124,22 @@ echo -e "\n▶️ Setting-up \"$SUITE\" test suite" 2>&1 | tee -a "$TEST_LOG" # allow overriding target if [ -z "$TARGET_WIKIBASE_UPGRADE_IMAGE_NAME" ]; then - export TARGET_WIKIBASE_UPGRADE_IMAGE_NAME="$WIKIBASE_IMAGE_NAME" + export TARGET_WIKIBASE_UPGRADE_IMAGE_NAME="$WIKIBASE_SUITE_WIKIBASE_IMAGE_URL" fi export WIKIBASE_TEST_IMAGE_NAME="$TARGET_WIKIBASE_UPGRADE_IMAGE_NAME:latest"; # echo "ℹ️ Target WIKIBASE_TEST_IMAGE_NAME is set to $WIKIBASE_TEST_IMAGE_NAME" if [ -n "$WDQS_SOURCE_IMAGE_NAME" ]; then - export WDQS_TEST_IMAGE_NAME="$WDQS_IMAGE_NAME:latest"; - docker load -i "../artifacts/$WDQS_IMAGE_NAME.docker.tar.gz" + export WDQS_TEST_IMAGE_NAME="$WIKIBASE_SUITE_WDQS_IMAGE_URL:latest"; + docker load -i "../artifacts/wdqs.docker.tar.gz" fi # load new version and start it echo "🔄 Creating Docker test services and volumes" 2>&1 | tee -a "$TEST_LOG" -docker load -i "../artifacts/$TARGET_WIKIBASE_UPGRADE_IMAGE_NAME.docker.tar.gz" >> $TEST_LOG 2>&1 -$TEST_COMPOSE -f suites/$SUITE_CONFIG_NAME/docker-compose.override.yml up -d --scale test-runner=0 >> $TEST_LOG 2>&1 +image_filename="../artifacts/$(echo "$TARGET_WIKIBASE_UPGRADE_IMAGE_NAME" | cut -d'/' -f2).docker.tar.gz" +docker load -i "$image_filename" +$TEST_COMPOSE -f suites/$SUITE_CONFIG_NAME/docker-compose.override.yml up -d --scale test-runner=0 $TEST_COMPOSE logs -f --no-color >> "$TEST_LOG" & # wait until containers start @@ -145,14 +147,14 @@ $TEST_COMPOSE logs -f --no-color >> "$TEST_LOG" & $TEST_COMPOSE run --rm test-runner -c suites/$SUITE_CONFIG_NAME/setup.sh # run update.php and log to separate file -echo -e "ℹ️ Running \"php /var/www/html/maintenance/update.php\"" 2>&1 | tee -a "$TEST_LOG" +echo -e "ℹ️ Running \"php /var/www/html/maintenance/update.php\"" -docker exec "$WIKIBASE_TEST_CONTAINER" php /var/www/html/maintenance/update.php --quick >> "$TEST_LOG" 2>&1 +docker exec "$WIKIBASE_TEST_CONTAINER" php /var/www/html/maintenance/update.php --quick -echo -e "\n✳️ Running \"$SUITE\" test suite" 2>&1 | tee -a "$TEST_LOG" +echo -e "\n✳️ Running \"$SUITE\" test suite" $TEST_COMPOSE run --rm test-runner -c "npm run test:run --silent" # shut down the stack, also remove volumes to test data does not interfere with next test runs -echo -e "🔄 Removing running Docker test services and volumes\n" 2>&1 | tee -a "$TEST_LOG" +echo -e "🔄 Removing running Docker test services and volumes\n" remove_services_and_volumes diff --git a/test/suites/docker-compose.yml b/test/suites/docker-compose.yml index 5db829cd7..3151953cd 100644 --- a/test/suites/docker-compose.yml +++ b/test/suites/docker-compose.yml @@ -72,7 +72,7 @@ services: - ./suites/wikibase-create-db.sql:/docker-entrypoint-initdb.d/01-create-db.sql wdqs-frontend: - image: "${WDQS_FRONTEND_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL}" restart: unless-stopped ports: # CONFIG - Change the 8282 here to expose the Query Service UI on a different port @@ -88,7 +88,7 @@ services: - WDQS_HOST=wdqs-proxy.svc wdqs: - image: "${WDQS_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_WDQS_IMAGE_URL}" restart: always command: /runBlazegraph.sh networks: @@ -103,7 +103,7 @@ services: - 9999 wdqs-proxy: - image: "${WDQS_PROXY_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL}" restart: unless-stopped environment: - PROXY_PASS_HOST=wdqs.svc:9999 @@ -117,7 +117,7 @@ services: - wdqs-proxy.svc wdqs-updater: - image: "${WDQS_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_WDQS_IMAGE_URL}" restart: unless-stopped command: /runUpdate.sh depends_on: diff --git a/test/suites/elasticsearch/docker-compose.override.yml b/test/suites/elasticsearch/docker-compose.override.yml index e4b18fe78..330f535c6 100644 --- a/test/suites/elasticsearch/docker-compose.override.yml +++ b/test/suites/elasticsearch/docker-compose.override.yml @@ -21,7 +21,7 @@ services: <<: *elastic_search elasticsearch: - image: "${ELASTICSEARCH_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL}" restart: unless-stopped networks: default: diff --git a/test/suites/quickstatements/docker-compose.override.yml b/test/suites/quickstatements/docker-compose.override.yml index 76ddfd8ef..fdf218b3d 100644 --- a/test/suites/quickstatements/docker-compose.override.yml +++ b/test/suites/quickstatements/docker-compose.override.yml @@ -45,7 +45,7 @@ services: - DB_NAME=client_wiki quickstatements: - image: "${QUICKSTATEMENTS_IMAGE_NAME}" + image: "${WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL}" restart: always volumes: - quickstatements-data:/quickstatements/data diff --git a/test/suites/upgrade/docker-compose.yml b/test/suites/upgrade/docker-compose.yml index 2d3a1f30b..f5151a629 100644 --- a/test/suites/upgrade/docker-compose.yml +++ b/test/suites/upgrade/docker-compose.yml @@ -29,7 +29,7 @@ services: - WIKIBASE_HOST mysql: - image: "${DEFAULT_DATABASE_IMAGE_NAME}" + image: "${MARIADB_IMAGE_URL}" restart: unless-stopped volumes: - mediawiki-mysql-data:/var/lib/mysql diff --git a/update_cache.sh b/update_cache.sh deleted file mode 100644 index 2bc78ec1f..000000000 --- a/update_cache.sh +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env bash -set -e - -function clone_if_not_present { - REPOSITORY_CACHE_NAME=$1 - GIT_URL=$2 - if [ ! -d "git_cache/$1" ]; then - git config --global --add safe.directory "$PWD/git_cache/$REPOSITORY_CACHE_NAME" - git clone --mirror "$GIT_URL" "git_cache/$REPOSITORY_CACHE_NAME" -q - fi -} - -function fetch_all { - REPOSITORY_CACHE_NAME=$1 - git config --global --add safe.directory "$PWD/git_cache/$REPOSITORY_CACHE_NAME" - cd "git_cache/$REPOSITORY_CACHE_NAME" - git fetch --all -q - cd - > /dev/null -} - -mkdir -p 'artifacts' -mkdir -p 'git_cache/skins/' -mkdir -p 'git_cache/services/' - -cd "$(dirname "$0")" - -skins=("Vector") -extensions=( \ - "Babel" \ - "CirrusSearch" \ - "cldr" \ - "ConfirmEdit" \ - "Elastica" \ - "EntitySchema" - "Nuke" \ - "OAuth" \ - "Scribunto" \ - "SyntaxHighlight_GeSHi" \ - "VisualEditor" \ - "UniversalLanguageSelector" - "Wikibase" \ - "WikibaseCirrusSearch" \ - "WikibaseManifest" \ - ) - -for arg in "$@"; do - - if [ "$arg" = "skins" ]; then - for skin in "${skins[@]}"; do - clone_if_not_present "skins/$skin.git" "https://gerrit.wikimedia.org/r/mediawiki/skins/$skin" - fetch_all "skins/$skin.git" - done - - elif [ "$arg" = "base" ]; then - clone_if_not_present "Wikibase.git" "https://gerrit.wikimedia.org/r/mediawiki/extensions/Wikibase" - fetch_all "Wikibase.git" - elif [ "$arg" = "bundle" ]; then - - # WikibaseLocalMedia - clone_if_not_present WikibaseLocalMedia.git https://github.com/ProfessionalWiki/WikibaseLocalMedia.git - fetch_all WikibaseLocalMedia.git - - # WikibaseEdtf - clone_if_not_present WikibaseEdtf.git https://github.com/ProfessionalWiki/WikibaseEdtf.git - fetch_all WikibaseEdtf.git - - for extension in "${extensions[@]}"; do - clone_if_not_present "$extension.git" "https://gerrit.wikimedia.org/r/mediawiki/extensions/$extension" - fetch_all "$extension.git" - done - elif [ "$arg" = "services" ]; then - # wdqs-frontend - clone_if_not_present "services/wikidata-query-gui.git" "https://gerrit.wikimedia.org/r/wikidata/query/gui" - fetch_all "services/wikidata-query-gui.git" - - # quickstatements - clone_if_not_present "services/quickstatements.git" https://github.com/magnusmanske/quickstatements - fetch_all "services/quickstatements.git" - - # magnus tools - clone_if_not_present "services/magnustools.git" https://bitbucket.org/magnusmanske/magnustools.git - fetch_all "services/magnustools.git" - - elif [ "$arg" = "core" ]; then - clone_if_not_present core.git "https://gerrit.wikimedia.org/r/mediawiki/core" - fetch_all "core.git" - fi -done diff --git a/variables.env b/variables.env index fa547ca23..b180775d9 100644 --- a/variables.env +++ b/variables.env @@ -15,50 +15,95 @@ RELEASE_VERSION=1.40.1 # and https://archiva.wikimedia.org/repository/releases/org/wikidata/query/rdf/service/ WDQS_VERSION=0.3.135 -# Head of master on 2023-11-09 -# as per https://github.com/wikimedia/wikidata-query-gui/ -WDQS_FRONTEND_COMMIT_HASH=eb1825b3431f3adf47cd774bf52344024e2f11e9 - -# Head of master on 2023-11-09 -# as per https://github.com/magnusmanske/quickstatements/commits/master -QUICKSTATEMENTS_COMMIT_HASH=c4b2c6b086b319aa32dcdd7a323edf188faaa66d - -# Head of master on 2023-11-09 -# as per https://bitbucket.org/magnusmanske/magnustools/commits/branch/master -MAGNUSTOOLS_COMMIT_HASH=e4abdb6b3bedfdde0325df513b272e1972b1af36 - -# Head of master on 2023-11-15 (including fix for deprecation warning on 1.40) -# as per https://github.com/ProfessionalWiki/WikibaseLocalMedia/tree/master -WIKIBASELOCALMEDIA_COMMIT_HASH=74c3f70893ffd76feefa9f1e3d77052db20ec3ac - -# Head of master on 2023-11-09 -# as per https://github.com/ProfessionalWiki/WikibaseEdtf/tree/master -WIKIBASEEDTF_COMMIT_HASH=5596403a27d8e4a5ac044257c49be3e0685318a6 - - -# No new extensions to be added -BUNDLE_EXT_EXTENSIONS=WikibaseLocalMedia,WikibaseEdtf -BUNDLE_WMF_EXTENSIONS=Babel,cldr,CirrusSearch,ConfirmEdit,Elastica,EntitySchema,Nuke,OAuth,Scribunto,SyntaxHighlight_GeSHi,UniversalLanguageSelector,VisualEditor,WikibaseCirrusSearch,WikibaseManifest - # NOTE ELASTICSEARCH_VERSION 7.10.3+ is release under a non-osi approved license # 7.10.2 is currently being used by the WMF 20230223 ELASTICSEARCH_VERSION=7.10.2 -ELASTICSEARCH_PLUGIN_EXTRA_VERSION=7.10.2-wmf4 - -COMPOSER_IMAGE_NAME=docker-registry.wikimedia.org/releng/composer-php82 -COMPOSER_IMAGE_VERSION=latest - -# these never need to regularly change but are for for DRY -WIKIBASE_IMAGE_NAME=wikibase -WDQS_IMAGE_NAME=wdqs -WDQS_FRONTEND_IMAGE_NAME=wdqs-frontend -WDQS_PROXY_IMAGE_NAME=wdqs-proxy -ELASTICSEARCH_IMAGE_NAME=elasticsearch -WIKIBASE_BUNDLE_IMAGE_NAME=wikibase-bundle -QUICKSTATEMENTS_IMAGE_NAME=quickstatements - -MEDIAWIKI_IMAGE_NAME=mediawiki -MEDIAWIKI_SETTINGS_TEMPLATE_FILE=LocalSettings.php.template +ELASTICSEARCH_IMAGE_URL=docker.elastic.co/elasticsearch/elasticsearch:7.10.2 + +# https://central.sonatype.com/artifact/org.wikimedia.search/extra +ELASTICSEARCH_PLUGIN_WIKIMEDIA_EXTRA=7.10.2-wmf4 + +# https://central.sonatype.com/artifact/org.wikimedia.search.highlighter/experimental-highlighter-elasticsearch-plugin +ELASTICSEARCH_PLUGIN_WIKIMEDIA_HIGHLIGHTER=7.10.2 + +# https://docker-registry.wikimedia.org/releng/composer-php82/tags/ +COMPOSER_IMAGE_URL=docker-registry.wikimedia.org/releng/composer-php82:0.1.0-s3 + +# https://hub.docker.com/_/mediawiki +MEDIAWIKI_IMAGE_URL=mediawiki:1.40.1 + +# https://hub.docker.com/_/mariadb +MARIADB_IMAGE_URL=mariadb:10.9 + +# https://hub.docker.com/_/nginx +NGINX_IMAGE_URL=nginx:1.25.3-bookworm + +# https://hub.docker.com/_/node +# TODO: move to debian for consistency +# TODO: upgrade, end of life +NODE_IMAGE_URL=node:14.21.3-alpine3.17 + +# https://hub.docker.com/_/php +PHP_IMAGE_URL=php:8.1.25-apache-bookworm + +# https://hub.docker.com/_/openjdk/ +# TODO: move to debian for consistency +# TODO: upgrade, end of life +JDK_IMAGE_URL=openjdk:8-jdk-alpine + +# https://hub.docker.com/_/debian +DEBIAN_IMAGE_URL=debian:bookworm-slim + +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Wikibase/+/refs/heads/REL1_40 +WIKIBASE_COMMIT=7a9b1f09f46e5feca3596168fdbcbeb25100c624 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Babel/+/refs/heads/REL1_40 +BABEL_COMMIT=1e1081ed52fedee17cef7be838930d50964cb2a9 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/cldr/+/refs/heads/REL1_40 +CLDR_COMMIT=2508ca06ff0dfe8ef4aa247f587f079f70db385e +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/CirrusSearch/+/refs/heads/REL1_40 +CIRRUSSEARCH_COMMIT=63130bc16b1cfb59659f0726784d37dd796f2a32 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/ConfirmEdit/+/refs/heads/REL1_40 +CONFIRMEDIT_COMMIT=dd1f631e222162fe22138cd8d981ffd4069157d2 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Elastica/+/refs/heads/REL1_40 +ELASTICA_COMMIT=7bffadfb85b51c7e0755c16cc950a330a9151fb4 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/EntitySchema/+/refs/heads/REL1_40 +ENTITYSCHEMA_COMMIT=0badbcd9e6d566bbb6356e7a621ac0887320b462 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Nuke/+/refs/heads/REL1_40 +NUKE_COMMIT=089c29c96e0818cb9391c30e3c127b6fb5fcdff5 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/OAuth/+/refs/heads/REL1_40 +OAUTH_COMMIT=fb2f037b73b4c873528cc426c4165038d5276727 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Scribunto/+/refs/heads/REL1_40 +SCRIBUNTO_COMMIT=cdecb51ccfda77a8ab05a2b4663336ebf91d2dc6 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/SyntaxHighlight_GeSHi/+/refs/heads/REL1_40 +SYNTAXHIGHLIGHT_GESHI_COMMIT=d0cf937d6a39431ff113d708dab08cb496ff8335 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/UniversalLanguageSelector/+/refs/heads/REL1_40 +UNIVERSALLANGUAGESELECTOR_COMMIT=51ea41a7066d51dcd245c6fa47fe47c71ee5189a +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/VisualEditor/+/refs/heads/REL1_40 +VISUALEDITOR_COMMIT=58d7d809829b4e326f1b8a0b69f83bae2ab851f7 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseCirrusSearch/+/refs/heads/REL1_40 +WIKIBASECIRRUSSEARCH_COMMIT=0e3e82e125254e86e262e6620c53e07ddf75c888 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseManifest/+/refs/heads/REL1_40 +WIKIBASEMANIFEST_COMMIT=10f0ddcb7eac28f56999f983f169cf7b1c6e240a +# https://github.com/ProfessionalWiki/WikibaseLocalMedia/tree/master +WIKIBASELOCALMEDIA_COMMIT=74c3f70893ffd76feefa9f1e3d77052db20ec3ac +# https://github.com/ProfessionalWiki/WikibaseEdtf/tree/master +WIKIBASEEDTF_COMMIT=5596403a27d8e4a5ac044257c49be3e0685318a6 +# https://github.com/magnusmanske/quickstatements/commits/master +QUICKSTATEMENTS_COMMIT=c4b2c6b086b319aa32dcdd7a323edf188faaa66d +# https://bitbucket.org/magnusmanske/magnustools/commits/branch/master +MAGNUSTOOLS_COMMIT=e4abdb6b3bedfdde0325df513b272e1972b1af36 +# https://gerrit.wikimedia.org/r/wikidata/query/gui +WDQSQUERYGUI_COMMIT=eb1825b3431f3adf47cd774bf52344024e2f11e9 + +# Image URLs for the docker images built by us. +# "wikibase/" is our prefix at docker hub, our default distribution point. +WIKIBASE_SUITE_WIKIBASE_IMAGE_URL=wikibase/wikibase +WIKIBASE_SUITE_WDQS_IMAGE_URL=wikibase/wdqs +WIKIBASE_SUITE_WDQS_FRONTEND_IMAGE_URL=wikibase/wdqs-frontend +WIKIBASE_SUITE_WDQS_PROXY_IMAGE_URL=wikibase/wdqs-proxy +WIKIBASE_SUITE_ELASTICSEARCH_IMAGE_URL=wikibase/elasticsearch +WIKIBASE_SUITE_WIKIBASE_BUNDLE_IMAGE_URL=wikibase/wikibase-bundle +WIKIBASE_SUITE_QUICKSTATEMENTS_IMAGE_URL=wikibase/quickstatements # Releasing tarballs RELEASE_DIR=/srv/org/wikimedia/releases/wikibase @@ -71,7 +116,10 @@ DOCKER_HUB_REPOSITORY_NAME=wikibase GZIP_COMPRESSION_RATE=9 -DEFAULT_DATABASE_IMAGE_NAME=mariadb:10.9 - -DEFAULT_BUNDLE_WMF_EXTENSIONS=Babel,cldr,CirrusSearch,ConfirmEdit,Elastica,EntitySchema,Nuke,OAuth,Scribunto,UniversalLanguageSelector,VisualEditor,WikibaseCirrusSearch,WikibaseManifest -DEFAULT_BUNDLE_EXT_EXTENSIONS=WikibaseLocalMedia +# wikibase default env +MW_SITE_NAME=wikibase-docker +MW_SITE_LANG=en +MW_WG_JOB_RUN_RATE=2 +MW_WG_ENABLE_UPLOADS=false +MW_WG_UPLOAD_DIRECTORY=/var/www/html/images +WIKIBASE_PINGBACK=false