diff --git a/.github/workflows/build_publish_image_release.yml b/.github/workflows/build_publish_image_release.yml index 2351b1ea2..3d24ef4de 100644 --- a/.github/workflows/build_publish_image_release.yml +++ b/.github/workflows/build_publish_image_release.yml @@ -9,6 +9,7 @@ on: - 'wdqs-frontend@*' - 'wdqs-proxy@*' - 'wikibase@*' + - 'wikibase-lts@*' permissions: contents: write diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml index 472c9e3b7..fb38aa5f0 100644 --- a/.github/workflows/create_release.yml +++ b/.github/workflows/create_release.yml @@ -10,12 +10,14 @@ on: default: 'wikibase' options: - deploy + - deploy-lts - elasticsearch - quickstatements - wdqs - wdqs-frontend - wdqs-proxy - wikibase + - wikibase-lts - All projects with unreleased changes dry_run: description: "Dry run, don't do it yet." diff --git a/build/build.sh b/build/build.sh index 3ca79ac4f..a18aae920 100755 --- a/build/build.sh +++ b/build/build.sh @@ -71,9 +71,12 @@ else ) fi -# transform TAGS to build args IMAGE_NAME=$(jq -r '.name' package.json) +# Removes "-lts" from end of name if it exists. +# LTS products are released on Dockerhub on the main image name +IMAGE_NAME=${IMAGE_NAME%-lts} + # publish to Dockerhub if [ "$PUBLISH" == true ]; then # IMAGE_REGISTRY implies dockerhub if empty diff --git a/build/wikibase-lts/CHANGELOG.md b/build/wikibase-lts/CHANGELOG.md new file mode 100644 index 000000000..c87bb3797 --- /dev/null +++ b/build/wikibase-lts/CHANGELOG.md @@ -0,0 +1,18 @@ +## 1.0.1 (2025-01-06) + + +### 🏡 Chore + +- bump mediawiki to 1.42.3, bump extensions + +## (from **wikibase@3.0.1** (2024-10-09)) + +### 📖 Documentation + +- Link to MediaWiki bundled extensions +- Switch from `.example.com` to `.example` + +### 🏡 Chore + +- Removes re-installation of extensions already packaged with MediaWiki (ConfirmEdit, Nuke, Scribunto, SyntaxHighlight_Geshi, VisualEditor) +- Updates Wikibase EDTF extension diff --git a/build/wikibase-lts/Dockerfile b/build/wikibase-lts/Dockerfile new file mode 100644 index 000000000..c0a4ff32a --- /dev/null +++ b/build/wikibase-lts/Dockerfile @@ -0,0 +1,212 @@ +# Build configuration +ARG PHP_IMAGE_URL +ARG COMPOSER_IMAGE_URL + +# ########################################################################### +# Based on https://github.com/wikimedia/mediawiki-docker/blob/1161796f04d6a6bcbec9fb4c67a8ce7248392403/1.41/apache/Dockerfile +# hadolint ignore=DL3006 +FROM ${PHP_IMAGE_URL} as mediawiki + +SHELL ["/bin/bash", "-exu", "-c"] + +# System dependencies +RUN apt-get update; \ + apt-get install -y --no-install-recommends \ + git \ + gettext-base \ + librsvg2-bin \ + imagemagick \ + # Required for SyntaxHighlighting + python3 \ + # Required for Scribunto + lua5.1 \ + ; \ + rm -rf /var/lib/apt/lists/* + +# Install the PHP extensions we need +# hadolint ignore=DL4006 +RUN savedAptMark="$(apt-mark showmanual)"; \ + \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + libicu-dev \ + libonig-dev \ + libbz2-dev=1.* \ + ; \ + \ + docker-php-ext-install -j "$(nproc)" \ + bz2 \ + calendar \ + intl \ + mbstring \ + mysqli \ + opcache \ + ; \ + \ + pecl install APCu-5.1.21; \ + docker-php-ext-enable \ + apcu \ + ; \ + rm -r /tmp/pear; \ + \ + # reset apt-mark's "manual" list so that "purge --auto-remove" will remove all build dependencies + apt-mark auto '.*' > /dev/null; \ + apt-mark manual $savedAptMark; \ + ldd "$(php -r 'echo ini_get("extension_dir");')"/*.so \ + | awk '/=>/ { print $3 }' \ + | sort -u \ + | xargs -r dpkg-query -S \ + | cut -d: -f1 \ + | sort -u \ + | xargs -rt apt-mark manual; \ + \ + apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ + rm -rf /var/lib/apt/lists/* + +# Enable Short URLs +RUN a2enmod rewrite; \ + { \ + echo ""; \ + echo " RewriteEngine On"; \ + echo " RewriteCond %{REQUEST_FILENAME} !-f"; \ + echo " RewriteCond %{REQUEST_FILENAME} !-d"; \ + echo " RewriteRule ^ %{DOCUMENT_ROOT}/index.php [L]"; \ + echo ""; \ + } > "$APACHE_CONFDIR/conf-available/short-url.conf"; \ + a2enconf short-url + +# Enable AllowEncodedSlashes for VisualEditor +RUN sed -i "s/<\/VirtualHost>/\tAllowEncodedSlashes NoDecode\n<\/VirtualHost>/" "$APACHE_CONFDIR/sites-available/000-default.conf" + +# set recommended PHP.ini settings +# see https://secure.php.net/manual/en/opcache.installation.php +RUN { \ + echo 'opcache.memory_consumption=128'; \ + echo 'opcache.interned_strings_buffer=8'; \ + echo 'opcache.max_accelerated_files=4000'; \ + echo 'opcache.revalidate_freq=60'; \ + } > /usr/local/etc/php/conf.d/opcache-recommended.ini + +# MediaWiki setup +WORKDIR /var/www/html +ARG MEDIAWIKI_VERSION +# hadolint ignore=DL4006 +RUN set -eux; \ + curl -fSL "https://releases.wikimedia.org/mediawiki/$(echo ${MEDIAWIKI_VERSION} | cut -d. -f1,2)/mediawiki-${MEDIAWIKI_VERSION}.tar.gz" -o mediawiki.tar.gz; \ + tar -x --strip-components=1 -f mediawiki.tar.gz; \ + rm -r mediawiki.tar.gz; \ + install -d /var/log/mediawiki -o www-data + +# ########################################################################### +# hadolint ignore=DL3006 +FROM ${COMPOSER_IMAGE_URL} as composer + +COPY --from=mediawiki --chown=nobody:nogroup /var/www/html /var/www/html +WORKDIR /var/www/html + +COPY composer.local.json composer.local.json + +# WORKAROUND for https://phabricator.wikimedia.org/T372458 +# Take wikibase submodules from github as phabricator rate limits us +COPY --chown=nobody:nogroup --chmod=755 \ + wikibase-submodules-from-github-instead-of-phabricator.patch \ + /tmp/wikibase-submodules-from-github-instead-of-phabricator.patch +USER root +RUN apt-get update; \ + apt-get install -y --no-install-recommends \ + patch \ + ; \ + rm -rf /var/lib/apt/lists/* +USER nobody + +ARG WIKIBASE_COMMIT +SHELL ["/bin/bash", "-euo", "pipefail", "-c"] +RUN set -x; \ + git clone https://gerrit.wikimedia.org/r/mediawiki/extensions/Wikibase /var/www/html/extensions/Wikibase && \ + git -C /var/www/html/extensions/Wikibase checkout ${WIKIBASE_COMMIT} && \ + patch -d /var/www/html/extensions/Wikibase -Np1 /usr/local/etc/php/conf.d/error_reporting.ini + +COPY --from=composer --chown=nobody:nogroup /var/www/html /var/www/html + +# Remove world writable flag, in combination with sticky bit it breaks the w link +# https://github.com/wmde/wikibase-release-pipeline/commit/545c7aeec8d0245dc597d500afc934b40e656b3c +# Make upload path writable for the webserver user +RUN chmod o-w /var/www/html && \ + ln -s /var/www/html/ /var/www/html/w && \ + chown www-data /var/www/html/images -R + +COPY wikibase-php.ini /usr/local/etc/php/conf.d/wikibase-php.ini +COPY entrypoint.sh /entrypoint.sh +COPY jobrunner-entrypoint.sh /jobrunner-entrypoint.sh +COPY htaccess /var/www/html/.htaccess +COPY LocalSettings.d LocalSettings.d +COPY default-extra-install.sh /default-extra-install.sh +COPY oauth.ini /templates/oauth.ini +COPY LocalSettings.wbs.php /templates/LocalSettings.wbs.php + +ENV DB_NAME=my_wiki \ + MW_WG_SITENAME=wikibase \ + MW_WG_LANGUAGE_CODE=en \ + ELASTICSEARCH_PORT=9200 + +ENTRYPOINT ["/bin/bash"] +CMD ["/entrypoint.sh"] diff --git a/build/wikibase-lts/LocalSettings.d/20_Wikibase.php b/build/wikibase-lts/LocalSettings.d/20_Wikibase.php new file mode 100644 index 000000000..7d83db2c3 --- /dev/null +++ b/build/wikibase-lts/LocalSettings.d/20_Wikibase.php @@ -0,0 +1,9 @@ + 'http://localhost' . $wgScriptPath . '/rest.php', +); diff --git a/build/wikibase-lts/LocalSettings.d/40_cldr.php b/build/wikibase-lts/LocalSettings.d/40_cldr.php new file mode 100644 index 000000000..4947122e6 --- /dev/null +++ b/build/wikibase-lts/LocalSettings.d/40_cldr.php @@ -0,0 +1,5 @@ + '/var/log/mediawiki/mw.resourceloader.log', + 'exception' => '/var/log/mediawiki/mw.exception.log', + 'error' => '/var/log/mediawiki/mw.error.log', + 'fatal' => '/var/log/mediawiki/mw.fatal.log', +); +$wgDebugLogFile = '/var/log/mediawiki/mw.debug.log'; + +$wgArticlePath = "/wiki/$1"; + +# Add configuration values here or above which should be set before extensions are loaded + +# Load extensions if present, alphabetically ordered by filename +foreach (glob("LocalSettings.d/*.php") as $filename) +{ + include $filename; +} + +############################################################################## +# End of generated LocalSettings.php +############################################################################## + +# Add configuration values below which should be set after extensions are loaded diff --git a/build/wikibase-lts/README.md b/build/wikibase-lts/README.md new file mode 100644 index 000000000..df9c72b8a --- /dev/null +++ b/build/wikibase-lts/README.md @@ -0,0 +1,173 @@ +# Wikibase Suite Wikibase Image + +[Wikibase](https://www.mediawiki.org/wiki/Wikibase) is a MediaWiki extension for working with versioned, semi-structured data in a central repository. + +This image contains the Wikibase extension running on top of MediaWiki. Wikibase and several other extensions are bundled in addition to [those hipped by MediaWiki](https://www.mediawiki.org/wiki/Bundled_extensions_and_skins). The MediaWiki application runs on top of PHP on an Apache web server in a Debian base image. + +> 💡 This image is part of Wikibase Suite (WBS). [WBS Deploy](https://github.com/wmde/wikibase-release-pipeline/deploy/README.md) provides everything you need to self-host a Wikibase instance out of the box. + +| Bundled Extension | Description | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| [Babel](https://www.mediawiki.org/wiki/Extension:Babel) | Adds a parser function to inform other users about language proficiency and categorize users of the same levels and languages. | +| [CLDR](https://www.mediawiki.org/wiki/Extension:CLDR) | Provides functions to localize the names of languages, countries, currencies, and time units based on their language code. | +| [Elastica](https://www.mediawiki.org/wiki/Extension:Elastica), [CirrusSearch](https://www.mediawiki.org/wiki/Extension:CirrusSearch), [WikibaseCirrusSearch](https://www.mediawiki.org/wiki/Extension:WikibaseCirrusSearch) | Elasticsearch integration for MediaWiki and Wikibase. | +| [EntitySchema](https://www.mediawiki.org/wiki/Extension:EntitySchema) | Allows to store Shape Expression Schemas on wiki pages. | +| [OAuth](https://www.mediawiki.org/wiki/Extension:OAuth) | Allow users to safely authorize another application ("consumer") to use the MediaWiki action API on their behalf. | +| [UniversalLanguageSelector](https://www.mediawiki.org/wiki/Extension:UniversalLanguageSelector) | Tool that allows users to select a language and configure its support in an easy way. | +| [WikibaseEdtf](https://github.com/ProfessionalWiki/WikibaseEdtf) | Adds support for the Extended Date/Time Format (EDTF) Specification via a new data type. | +| [WikibaseLocalMedia](https://github.com/ProfessionalWiki/WikibaseLocalMedia) | Adds support for local media files to Wikibase via a new data type. | +| [WikibaseManifest](https://www.mediawiki.org/wiki/Extension:WikibaseManifest) | API-provided metadata for structured data repository. | + +## Requirements + +In order to run Wikibase, you need: + +- Database +- Configuration volume +- Initial settings via environment variables +- Job runner + +### Database + +This is the database MediaWiki will connect to and store all its data in. Technically, MediaWiki supports multiple database engines, but MariaDB is the most commonly used. This is also the only engine used to test the image before release. + +### Configuration volume + +MediaWiki will generate a `LocalSettings.php` file on first launch. Once this file has been generated, you own and control it. This file is stored in the configuration volume. + +### Environment variables for initial settings + +These variables are only respected on first launch in order to generate MediaWiki's `LocalSettings.php` file. When launching the image with a `LocalSettings.php` file present in the configuration volume, environment variables will not be taken into account. + +Variables in **bold** are required on first launch without `LocalSettings.php` in the configuration volume. The image will fail to start if one of those variables does not have a value. Default values do not need to be overwritten. + +| Variable | Default | Description | +| ---------------------------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **`DB_SERVER`** | undefined | Hostname and port for the MySQL server to use for MediaWiki & Wikibase | +| **`DB_USER`** | undefined | Username to use for the MySQL server | +| **`DB_PASS`** | undefined | Password to use for the MySQL server | +| **`DB_NAME`** | "my_wiki" | Database name to use for the MySQL server | +| **`MW_ADMIN_NAME`** | undefined | Admin username to create on MediaWiki first install | +| **`MW_ADMIN_PASS`** | undefined | Admin password to use for admin account on first install | +| **`MW_ADMIN_EMAIL`** | undefined | Admin password to use for admin account on first install | +| **`MW_WG_SERVER`** | undefined | `$wgServer` to use for MediaWiki. A value matching how this site is accessed from the user's browser is required. | +| **`MW_WG_SITENAME`** | "wikibase" | `$wgSitename` to use for MediaWiki | +| **`MW_WG_LANGUAGE_CODE`** | "en" | `$wgLanguageCode` to use for MediaWiki | +| `ELASTICSEARCH_HOST` | undefined | Hostname of an Elasticsearch server with the Wikibase extension installed, such as [wikibase/elasticsearch](https://hub.docker.com/r/wikibase/elasticsearch). Leave this undefined to disable Elasticsearch. | +| `ELASTICSEARCH_PORT` | 9200 | Port on which Elasticsearch is available | +| `QUICKSTATEMENTS_PUBLIC_URL` | undefined | Public URL of the QuickStatements server, such as [wikibase/quickstatements](https://hub.docker.com/r/wikibase/quickstatements). Leave undefined to disable QuickStatements functionality. | + +### Job runner + +MediaWiki/Wikibase depends on [jobs being run in the background](https://www.mediawiki.org/wiki/Manual:Job_queue). This can be either done on HTTP request or by a dedicated job runner. The default configuration of this image requires an external job runner like this. + +To set up an external job runner, use this image for a second container, overwrite the command to `/jobrunner-entrypoint.sh` and share the same configuration volume with it. + +## Example + +Here's an example of how to run this image together with the [WBS Wikibase image](https://hub.docker.com/r/wikibase/wikibase) using Docker Compose. + +```yml +services: + wikibase: + image: wikibase/wikibase + ports: + - 80:80 + volumes: + - ./config:/config + - wikibase-image-data:/var/www/html/images + environment: + MW_ADMIN_NAME: "admin" + MW_ADMIN_PASS: "change-this-password" + MW_ADMIN_EMAIL: "admin@wikibase.example" + MW_WG_SERVER: http://localhost + DB_SERVER: mysql:3306 + DB_NAME: "my_wiki" + DB_USER: "mariadb-user" + DB_PASS: "change-this-password" + healthcheck: + test: curl --silent --fail localhost/wiki/Main_Page + interval: 10s + start_period: 5m + depends_on: + mysql: + condition: service_healthy + restart: unless-stopped + + wikibase-jobrunner: + image: wikibase/wikibase + volumes_from: + - wikibase + command: /jobrunner-entrypoint.sh + depends_on: + wikibase: + condition: service_healthy + restart: always + + mysql: + image: mariadb:10.11 + volumes: + - mysql-data:/var/lib/mysql + environment: + MYSQL_DATABASE: "my_wiki" + MYSQL_USER: "mariadb-user" + MYSQL_PASSWORD: "change-this-password" + MYSQL_RANDOM_ROOT_PASSWORD: yes + healthcheck: + test: healthcheck.sh --connect --innodb_initialized + start_period: 1m + interval: 20s + timeout: 5s + restart: unless-stopped + +volumes: + wikibase-image-data: + mysql-data: +``` + +## Releases + +Official releases of this image can be found on [Docker Hub wikibase/wikibase](https://hub.docker.com/r/wikibase/wikibase). + +## Tags and versioning + +This Wikibase image is using [semantic versioning](https://semver.org/spec/v2.0.0.html). + +We provide several tags that relate to the versioning semantics. + +| Tag | Example | Description | +| ----------------------------------------------- | ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| _MAJOR_ | 3 | Tags the latest image with this major version. Gets overwritten whenever a new version is released with this major version. This will include new builds triggered by base image changes, patch version updates and minor version updates. | +| _MAJOR_._MINOR_ | 3.1 | Tags the latest image with this major and minor version. Gets overwritten whenever a new version is released with this major and minor version. This will include new builds triggered by base image changes and patch version updates. | +| _MAJOR_._MINOR_._PATCH_ | 3.1.7 | Tags the latest image with this major, minor and patch version. Gets overwritten whenever a new version is released with this major, minor and patch version. This only happens for new builds triggered by base image changes. | +| _MAJOR_._MINOR_._PATCH_\_mw*MW-VERSION* | 3.1.7_mw1.41.1 | Same as above, but also mentioning the current MediaWiki version. | +| _MAJOR_._MINOR_._PATCH_\_build*BUILD-TIMESTAMP* | 3.1.7_build20240530103941 | Tag that never gets overwritten. Every image will have this tag with a unique build timestamp. Can be used to reference images explicitly for reproducibility. | + +## Internal filesystem layout + +Hooking into the internal filesystem can extend the functionality of this image. + +| Directory | Description | +| ------------------------------- | -------------------------------------------------------------------------------------------------------------- | +| `/var/www/html` | Base MediaWiki directory | +| `/var/www/html/images` | MediaWiki image and media upload directory | +| `/var/www/html/skins` | MediaWiki skins directory | +| `/var/www/html/extensions` | MediaWiki extensions directory | +| `/var/www/html/LocalSettings.d` | MediaWiki LocalSettings configuration directory, sourced in alphabetical order at the end of LocalSettings.php | +| `/templates/` | Directory containing templates | + +| File | Description | +| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `/default-extra-install.sh` | Script for automatically creating Elasticsearch indices and creating OAuth consumer for QuickStatements | +| `/extra-install.sh` | Optional script for custom functionality to be ran with MediaWiki install (when generating LocalSettings.php) | +| `/templates/LocalSettings.wbs.php` | Wikibase-specific settings appended to the MediaWiki install generated `LocalSettings.php`. Specifically, this loads the Wikibase repo and client as well as all the other bundled extensions. | + +## Source + +This image is built from this [Dockerfile](https://github.com/wmde/wikibase-release-pipeline/blob/main/build/wikibase/Dockerfile). + +## Authors & contact + +This image is maintained by the Wikibase Suite Team at [Wikimedia Germany (WMDE)](https://wikimedia.de). + +If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/build/wikibase-lts/build.env b/build/wikibase-lts/build.env new file mode 100644 index 000000000..4dc5256b4 --- /dev/null +++ b/build/wikibase-lts/build.env @@ -0,0 +1,77 @@ + +# ############################################################################## +# Mediawiki version +# ############################################################################## +# Update only patch versions for security releases. +# Choose latest version for major releases. +# https://releases.wikimedia.org/mediawiki/ +MEDIAWIKI_VERSION=1.39.10 + +# ############################################################################## +# PHP Composer +# ############################################################################## +# Just used for building - PHP's NPM +# Typically, no need to update. +# https://docker-registry.wikimedia.org/releng/composer-php82/tags/ +COMPOSER_IMAGE_URL=docker-registry.wikimedia.org/releng/composer-php82:0.1.1-s2 + +# ############################################################################## +# Third party base images +# ############################################################################## +# Update only patch versions for security releases. +# Choose latest LTS version for major releases. + +# https://www.php.net/supported-versions.php +# https://hub.docker.com/_/php +PHP_IMAGE_URL=php:8.3.8-apache-bookworm + +# https://hub.docker.com/_/debian +DEBIAN_IMAGE_URL=debian:bookworm-slim + +# ****************************************************************************** +# ****************************************************************************** +# Versions below are automatically updated by ../update_commits +# ****************************************************************************** +# ****************************************************************************** + +# ############################################################################## +# WMF maintained extensions +# ############################################################################## +# Updated automatically by ../update_commits +# Versions in REL_ branches ensure compatibility with respective mediawiki versions. +# Shouldn't require much of a review. +# +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Wikibase/+/refs/heads/REL1_39 +WIKIBASE_COMMIT=2e2bd1f305f2c1e1bd77c55e78042a8e9ab50021 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Babel/+/refs/heads/REL1_39 +BABEL_COMMIT=76fcfe4d6a7d455418913a79ce0662bc4b7547a6 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/cldr/+/refs/heads/REL1_39 +CLDR_COMMIT=685ceb15653c647c6e325f46a1a46c171643e77f +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/CirrusSearch/+/refs/heads/REL1_39 +CIRRUSSEARCH_COMMIT=4549fb80ef28e2f994ad56a0101cb3d9c720beab +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/Elastica/+/refs/heads/REL1_39 +ELASTICA_COMMIT=13b91fc23a8e1b12a128bb129610820df91127c9 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/EntitySchema/+/refs/heads/REL1_39 +ENTITYSCHEMA_COMMIT=27468025286537622e286d84f4f43cd70a9ea0ff +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/OAuth/+/refs/heads/REL1_39 +OAUTH_COMMIT=414d8d45898174b537983e3757b380a4070fccc6 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/UniversalLanguageSelector/+/refs/heads/REL1_39 +UNIVERSALLANGUAGESELECTOR_COMMIT=5c3e5280b6b111ecc7bc9831b230370ab867ff18 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseCirrusSearch/+/refs/heads/REL1_39 +WIKIBASECIRRUSSEARCH_COMMIT=564f42559835ec069b7b6fa4bd3be01ec2db0f04 +# https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/extensions/WikibaseManifest/+/refs/heads/REL1_39 +WIKIBASEMANIFEST_COMMIT=a9acda8d9f3e671e30fdf14c00570d1b06a763ab + +# ############################################################################## +# Community maintained extensions +# ############################################################################## +# Updated automatically by ../update_commits +# Those extensions follow arbitrary versioning strategies. Their compatibility +# with mediawiki versions has to be checked explicitly. Review carefully. +# +# https://github.com/ProfessionalWiki/WikibaseLocalMedia/commits/master +WIKIBASELOCALMEDIA_COMMIT=77b2555dbf6cf8268e0077c6f38225d6833ee731 +# https://github.com/ProfessionalWiki/WikibaseEdtf/commits/master +WIKIBASEEDTF_COMMIT=8b5f88f1c6f8ac996db74df01bb4fafe7b1bba26 + +IMAGE_TAGS=( "mw${MEDIAWIKI_VERSION}" ) diff --git a/build/wikibase-lts/composer.local.json b/build/wikibase-lts/composer.local.json new file mode 100644 index 000000000..796ad0fe7 --- /dev/null +++ b/build/wikibase-lts/composer.local.json @@ -0,0 +1,7 @@ +{ + "extra": { + "merge-plugin": { + "include": [ "extensions/*/composer.json", "skins/*/composer.json" ] + } + } +} diff --git a/build/wikibase-lts/default-extra-install.sh b/build/wikibase-lts/default-extra-install.sh new file mode 100755 index 000000000..a1343987c --- /dev/null +++ b/build/wikibase-lts/default-extra-install.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -ex + +# Enables and configures elasticsearch index +if [ -z "${ELASTICSEARCH_HOST:-}" ] ; then + echo "Skipping Elasticsearch setup ..." +else + # shellcheck disable=2153 # do not warn about unused variables + php /var/www/html/extensions/CirrusSearch/maintenance/UpdateSearchIndexConfig.php + php /var/www/html/extensions/CirrusSearch/maintenance/ForceSearchIndex.php --skipParse + php /var/www/html/extensions/CirrusSearch/maintenance/ForceSearchIndex.php --skipLinks --indexOnSkip +fi + +# Creates an OAuth consumer for quickstatements +if [ -z "${QUICKSTATEMENTS_PUBLIC_URL:-}" ] ; then + echo "Skipping QuickStatements setup ..." +else + # Attempt to create OAuth consumer for QuickStatements + if php /var/www/html/extensions/OAuth/maintenance/createOAuthConsumer.php \ + --approve \ + --callbackUrl "$QUICKSTATEMENTS_PUBLIC_URL/api.php" \ + --callbackIsPrefix true --user "$MW_ADMIN_NAME" --name QuickStatements --description QuickStatements --version 1.0.1 \ + --grants createeditmovepage --grants editpage --grants highvolume --jsonOnSuccess > /quickstatements/data/qs-oauth.json; then + # Check if JSON file was created + if [[ -f /quickstatements/data/qs-oauth.json ]]; then + OAUTH_CONSUMER_KEY=$(grep -o '"key":"[^"]*' /quickstatements/data/qs-oauth.json | grep -o '[^"]*$') + OAUTH_CONSUMER_SECRET=$(grep -o '"secret":"[^"]*' /quickstatements/data/qs-oauth.json | grep -o '[^"]*$') + + export OAUTH_CONSUMER_KEY + export OAUTH_CONSUMER_SECRET + + envsubst < /templates/oauth.ini > /quickstatements/data/oauth.ini + fi + fi +fi diff --git a/build/wikibase-lts/dockerhub.md b/build/wikibase-lts/dockerhub.md new file mode 100644 index 000000000..de05de866 --- /dev/null +++ b/build/wikibase-lts/dockerhub.md @@ -0,0 +1,21 @@ +# Wikibase Suite wikibase Image + +Wikibase Suite (WBS) eases self-hosting [Wikibase](https://wikiba.se) in production, allowing you to maintain a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). + +If you want to host your own WBS instance, head over to the [WBS Deploy documentation](https://github.com/wmde/wikibase-release-pipeline/blob/main/deploy/README.md). + +# Documentation + +Version specific documentation for this image is hosted in our [git repository](https://github.com/wmde/wikibase-release-pipeline/). + +## Currently supported versions + +- July 2024 [3.x.x on MediaWiki 1.42](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-3/build/wikibase/README.md) +- July 2024 [2.x.x on MediaWiki 1.41](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-2/build/wikibase/README.md) +- July 2024 [1.x.x on MediaWiki 1.39](https://github.com/wmde/wikibase-release-pipeline/blob/deploy-1/build/wikibase/README.md) + +## Legacy versions + +- 17 April 2024 [1.41.1-wmde.20](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.20/build/wikibase/README.md) +- 17 April 2024 [1.40.3-wmde.19](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.19/build/wikibase/README.md) +- 29 April 2024 [1.39.7-wmde.18](https://github.com/wmde/wikibase-release-pipeline/blob/wmde.18/build/wikibase/README.md) diff --git a/build/wikibase-lts/entrypoint.sh b/build/wikibase-lts/entrypoint.sh new file mode 100644 index 000000000..0927d2378 --- /dev/null +++ b/build/wikibase-lts/entrypoint.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +# This file is provided by the wikibase/wikibase docker image. + +# Exit immediately with message if no /config volume is available +if [ ! -d "/config" ]; then + echo "A volume mapped to /config is required." + exit 1 +fi + +# Exit immediate on errors or unset variables from here onwards +set -eu + +# Take wikibase-php.ini from user config if present +if [ -e "/config/wikibase-php.ini" ]; then + cp /config/wikibase-php.ini /usr/local/etc/php/conf.d/wikibase-php.ini + +# Otherwise, make our stock wikibase-php.ini visible to the user for customization +else + cp /usr/local/etc/php/conf.d/wikibase-php.ini /config/wikibase-php.ini +fi + +if [ -e "/config/LocalSettings.php" ]; then + cp /config/LocalSettings.php /var/www/html/LocalSettings.php + # Always run update (this might be the first run off of a new image version on existing config and data) + # TODO: Switch to the new way of running maintenance scripts after dropping 1.39 support end of 2025 + #php /var/www/html/maintenance/run.php update --quick + php /var/www/html/maintenance/update.php --quick +else + echo "/config/LocalSettings.php not found, running MediaWiki install." + + # Check for required env vars + set +u + required_vars=( + DB_SERVER + DB_PASS + DB_USER + DB_NAME + MW_ADMIN_NAME + MW_ADMIN_EMAIL + MW_ADMIN_PASS + MW_WG_SERVER + MW_WG_LANGUAGE_CODE + MW_WG_SITENAME + ) + for var in "${required_vars[@]}"; do + if [ -z "${!var}" ]; then + echo "$var is required but isn't set. You should pass it to Docker. See: https://docs.docker.com/engine/reference/commandline/run/#set-environment-variables--e---env---env-file" + exit 1 + fi + done + set -u + + # Run MediaWiki install script, and update values + # TODO: Switch to the new way of running maintenance scripts after dropping 1.39 support end of 2025 + #php /var/www/html/maintenance/run.php install \ + php /var/www/html/maintenance/install.php \ + --server "$MW_WG_SERVER" \ + --scriptpath "/w" \ + --dbuser "$DB_USER" \ + --dbpass "$DB_PASS" \ + --dbname "$DB_NAME" \ + --dbserver "$DB_SERVER" \ + --pass "$MW_ADMIN_PASS" \ + --lang "$MW_WG_LANGUAGE_CODE" \ + "$MW_WG_SITENAME" \ + "$MW_ADMIN_NAME" + + # Include WBS customizations to generated LocalSettings.php + { + echo + echo '# Configuration added by Wikibase Suite installer in entrypoint.sh' + echo + if [[ -v ELASTICSEARCH_HOST ]]; then + echo "\$elasticsearchHost = '$ELASTICSEARCH_HOST';" + fi + echo + grep -v "> /var/www/html/LocalSettings.php + + # Replace /config/LocalSettings.php with newly generated LocalSettings.php + cp /var/www/html/LocalSettings.php /config/LocalSettings.php + # Update the MW Admin email address (if this admin user doesn't already exist, a new one will be created) + # TODO: Switch to the new way of running maintenance scripts after dropping 1.39 support end of 2025 + #php /var/www/html/maintenance/run.php resetUserEmail --no-reset-password "$MW_ADMIN_NAME" "$MW_ADMIN_EMAIL" + php /var/www/html/maintenance/resetUserEmail.php --no-reset-password "$MW_ADMIN_NAME" "$MW_ADMIN_EMAIL" + + # TODO: Switch to the new way of running maintenance scripts after dropping 1.39 support end of 2025 + #php /var/www/html/maintenance/run.php update --quick + php /var/www/html/maintenance/update.php --quick + + if [ -f /default-extra-install.sh ]; then + bash /default-extra-install.sh + fi + + if [ -f /extra-install.sh ]; then + bash /extra-install.sh + fi +fi + +# Run the actual entry point +docker-php-entrypoint apache2-foreground diff --git a/build/wikibase-lts/htaccess b/build/wikibase-lts/htaccess new file mode 100644 index 000000000..8401c6bc9 --- /dev/null +++ b/build/wikibase-lts/htaccess @@ -0,0 +1,15 @@ +# This file is provided by the wikibase/wikibase docker image. +## http://www.mediawiki.org/wiki/Manual:Short_URL/Apache + +# Enable the rewrite engine +RewriteEngine On + +# Short url for wiki pages +RewriteRule ^/?wiki(/.*)?$ %{DOCUMENT_ROOT}/w/index.php [L] + +# Redirect / to Main Page +RewriteRule ^/*$ %{DOCUMENT_ROOT}/w/index.php [L] + +# rewrite /entity/ URLs like wikidata per +# https://meta.wikimedia.org/wiki/Wikidata/Notes/URI_scheme +RewriteRule ^/?entity/(.*)$ /wiki/Special:EntityData/$1 [R=303,QSA] diff --git a/build/wikibase-lts/jobrunner-entrypoint.sh b/build/wikibase-lts/jobrunner-entrypoint.sh new file mode 100644 index 000000000..db4dc2de4 --- /dev/null +++ b/build/wikibase-lts/jobrunner-entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Originally inspired by Brennen Bearnes jobrunner entrypoint +# https://gerrit.wikimedia.org/r/plugins/gitiles/releng/dev-images/+/refs/heads/master/common/jobrunner/entrypoint.sh + +kill_runner() { + kill "$PID" 2> /dev/null +} +trap kill_runner SIGTERM + +while true; do + php maintenance/runJobs.php --wait --maxjobs="${JOBRUNNER_MAX_JOBS:-5}" --conf /config/LocalSettings.php & + PID=$! + wait "$PID" +done diff --git a/build/wikibase-lts/oauth.ini b/build/wikibase-lts/oauth.ini new file mode 100644 index 000000000..557d34adb --- /dev/null +++ b/build/wikibase-lts/oauth.ini @@ -0,0 +1,5 @@ +; HTTP User-Agent header +agent = 'Wikibase Docker QuickStatements' +; assigned by Special:OAuthConsumerRegistration (request modelled after https://www.wikidata.org/wiki/Special:OAuthListConsumers/view/77b4ae5506dd7dbb0bb07f80e3ae3ca9) +consumerKey = '${OAUTH_CONSUMER_KEY}' +consumerSecret = '${OAUTH_CONSUMER_SECRET}' diff --git a/build/wikibase-lts/package.json b/build/wikibase-lts/package.json new file mode 100644 index 000000000..fc00dbb72 --- /dev/null +++ b/build/wikibase-lts/package.json @@ -0,0 +1,12 @@ +{ + "name": "wikibase-lts", + "version": "1.0.0", + "nx": { + "targets": { + "lint": {}, + "build": {}, + "nx-release-publish": {}, + "update-commits": {} + } + } +} diff --git a/build/wikibase-lts/wikibase-php.ini b/build/wikibase-lts/wikibase-php.ini new file mode 100644 index 000000000..a6f5aac3c --- /dev/null +++ b/build/wikibase-lts/wikibase-php.ini @@ -0,0 +1,15 @@ +; ############################################################################## +; Wikibase Suite PHP settings +; ############################################################################## +; Feel free to adjust those settings according your needs. +; Changes will be picked up automatically by the wikibase container on restart. + +; Set the PHP upload limits be the same as the MediaWiki default of 100M. +; Should match the wgMaxUploadSize setting in LocalSettings.php +file_uploads = On +upload_max_filesize = 100M +post_max_size = 100M + +; Set the PHP memory and execution limits +memory_limit = 100M +max_execution_time = 600 diff --git a/build/wikibase-lts/wikibase-submodules-from-github-instead-of-phabricator.patch b/build/wikibase-lts/wikibase-submodules-from-github-instead-of-phabricator.patch new file mode 100644 index 000000000..f708808c1 --- /dev/null +++ b/build/wikibase-lts/wikibase-submodules-from-github-instead-of-phabricator.patch @@ -0,0 +1,21 @@ +diff --git a/.gitmodules b/.gitmodules +index df41c768af..e9926d6ddd 100644 +--- a/.gitmodules ++++ b/.gitmodules +@@ -3,13 +3,13 @@ + url = https://gerrit.wikimedia.org/r/data-values/value-view + [submodule "view/lib/wikibase-serialization"] + path = view/lib/wikibase-serialization +- url = https://phabricator.wikimedia.org/source/wikibase-serialization.git ++ url = https://github.com/wmde/WikibaseSerializationJavaScript.git + [submodule "view/lib/wikibase-data-values"] + path = view/lib/wikibase-data-values +- url = https://phabricator.wikimedia.org/source/datavalues-javascript.git ++ url = https://github.com/wmde/DataValuesJavaScript.git + [submodule "view/lib/wikibase-data-model"] + path = view/lib/wikibase-data-model +- url = https://phabricator.wikimedia.org/source/wikibase-data-model.git ++ url = https://github.com/wmde/WikibaseDataModelJavaScript.git + [submodule "view/lib/wikibase-termbox"] + path = view/lib/wikibase-termbox + url = https://gerrit.wikimedia.org/r/wikibase/termbox diff --git a/deploy-lts/.env b/deploy-lts/.env new file mode 100644 index 000000000..216749c26 --- /dev/null +++ b/deploy-lts/.env @@ -0,0 +1,37 @@ +# ############################################################################## +# Wikibase Suite Deploy - initial configuration +# +# This file is a template for the .env file used by the Wikibase Suite Deploy +# docker-compose setup. +# +# WARNING: Do not add comments on the same line as env vars: in some +# environments they will be included in the var! +# ############################################################################## + +# Public hostname configuration. +# These domain names for your Wikibase Suite services should be configured on +# your DNS server to point to the public IP address of the server being +# deployed to. Note that you need three distinct names, e.g. three different +# subdomains. Otherwise the reverse proxy cannot route properly. +WIKIBASE_PUBLIC_HOST=wikibase.example +WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend.example +QUICKSTATEMENTS_PUBLIC_HOST=quickstatements.example + +# MediaWiki / Wikibase user configuration. +# Password must be at least 10 characters, different from your username and not +# appear in the list of commonly used passwords this project uses. +MW_ADMIN_NAME=admin +MW_ADMIN_EMAIL=admin@wikibase.example +MW_ADMIN_PASS=change-this-password + +# MediaWiki / Wikibase database configuration. +# Those settings are used by the MariaDB container when creating a new +# database and by MediaWiki when generating a new LocalSettings.php. They will not +# be set on an existing database, nor will MediaWiki update those settings in +# your LocalSettings.php. To change those settings, either adjust them manually +# in MariaDB and your LocalSettings.php, or delete your MariaDB volume +# "mysql-data" and your LocalSettings.php from the ./config directory and +# restart. +DB_NAME=my_wiki +DB_USER=sqluser +DB_PASS=change-this-password diff --git a/deploy-lts/CHANGELOG.md b/deploy-lts/CHANGELOG.md new file mode 100644 index 000000000..4a1973532 --- /dev/null +++ b/deploy-lts/CHANGELOG.md @@ -0,0 +1,43 @@ +## 1.0.1 (2025-01-06) + + +### 🩹 Fixes + +- bump wdqs-updater version to 2 as well + +## (from Deploy 3.0.2 (2024-10-28)) + + +### 🩹 Fixes + +- provide concept uri to wdqs + + +### 📖 Documentation + +- WBS specific usage of WDQS frontend + +- **deploy:** fix backup script volume names + + +### 🏡 Chore + +- bump deploy to version 2 + +## (from **deploy@3.0.1** (2024-10-09)) + +### 🩹 Fixes + +- `*_PUBLIC_HOST` URLs default to `.example` TLD from `example.com` ([T372455](https://phabricator.wikimedia.org/T372455)) +- Removes published ports for wikibase, quickstatements, and wdqs-frontend services ([T372455](https://phabricator.wikimedia.org/T372455)) +- Wikibase waits for elasticsearch service to be up ([T371162](https://phabricator.wikimedia.org/T371162)) + +### 📖 Documentation + +- Do not use `--wait` in `docker compose up` in order to see logs +- Link to WDQS updater crash FAQ entry +- Corrected typos + +### 🏡 Chore + +- Adds default Traefik LetsEncrypt configuration for services diff --git a/deploy-lts/README.md b/deploy-lts/README.md new file mode 100644 index 000000000..9abeb66bb --- /dev/null +++ b/deploy-lts/README.md @@ -0,0 +1,347 @@ +# Wikibase Suite Deploy + +Wikibase Suite (WBS) Deploy is a containerized, production-ready [Wikibase](https://wikiba.se) system that allows you to self-host a knowledge graph similar to [Wikidata](https://www.wikidata.org/wiki/Wikidata:Main_Page). In addition to Wikibase on MediaWiki, WBS Deploy includes the Wikidata Query Service (WDQS), QuickStatements, Elasticsearch, and a Traefik reverse proxy with SSL termination and ACME support. The service orchestration is implemented using Docker Compose. + +> 🔧 This document is for people wanting to self-host the full Wikibase Suite using Wikibase Suite Deploy. If you are looking for individual WBS images, head over to [hub.docker.com/u/wikibase](https://hub.docker.com/u/wikibase). + +> 💡 This document presumes familiarity with basic Linux administration tasks and with Docker and Docker Compose. + +## What's in the box? + +WBS Deploy consists of the following services: + +- **[Wikibase](https://hub.docker.com/r/wikibase/wikibase)** MediaWiki packaged with the Wikibase extension and other commonly used extensions. +- **Job Runner** The MediaWiki [JobRunner](https://www.mediawiki.org/wiki/Manual:Job_queue#Cron) service which uses the same Wikibase container as above. +- **[MariaDB](https://hub.docker.com/_/mariadb)** Database service for MediaWiki and Wikibase. +- **[Elasticsearch](https://hub.docker.com/r/wikibase/elasticsearch)** Search service used by MediaWiki. +- **[WDQS](https://hub.docker.com/r/wikibase/wdqs)** Wikidata Query Service to process SPARQL queries. +- **[WDQS Frontend](https://hub.docker.com/r/wikibase/wdqs-frontend)** Web front end for SPARQL queries. +- **[WDQS Proxy](https://hub.docker.com/r/wikibase/wdqs-proxy)** A middle layer for WDQS which serves to filter requests and make the service more secure. +- **[WDQS Updater](https://www.mediawiki.org/wiki/Wikidata_Query_Service/User_Manual#runUpdate.sh)** Keeps the WDQS data in sync with Wikibase. +- **[Quickstatements](https://hub.docker.com/r/wikibase/quickstatements)** A web-based tool to import and manipulate large amounts of data. +- **[Traefik](https://hub.docker.com/_/traefik)** A reverse proxy that handles TLS termination and SSL certificate renewal through ACME. + +## Quickstart + +> 💡 If you want to run a quick test on a machine that has no public IP address (such as your local machine), check our [FAQ entry](#can-i-host-wbs-deploy-locally) below. + +### Requirements + +#### Hardware + +- Network connection with a public IP address +- AMD64 architecture +- 8 GB RAM +- 4 GB free disk space + +#### Software + +- Docker 22.0 (or greater) +- Docker Compose 2.10 (or greater) +- git + +#### Domain names + +You need three DNS records that resolve to your machine's IP address, one for each user-facing service: + +- Wikibase, e.g., "wikibase.example" +- QueryService, e.g., "query.example" +- QuickStatements, e.g., "quickstatements.example" + +### Initial setup + +#### Download WBS Deploy + +Check out the files from Github, move to the subdirectory `deploy` and check out the latest stable branch. + +```sh +git clone https://github.com/wmde/wikibase-release-pipeline +cd wikibase-release-pipeline/deploy +git checkout deploy-3 +``` + +#### Initial configuration + +Make a copy of the [configuration template](./template.env) in the `wikibase-release-pipeline/deploy` directory. + +```sh +cp template.env .env +``` + +Follow the instructions in the comments in your newly created `.env` file to set usernames, passwords and domain names. + +#### Starting + +Run the following command from within `wikibase-release-pipeline/deploy`: + +```sh +docker compose up +``` + +The first start can take a couple of minutes. Wait for your shell prompt to return. + +🎉 Congratulations, your Wikibase Suite instance should now be up and running. Web interfaces are available over HTTPS (port 443) for the domain names you configured for Wikibase, the WDQS front end and Quickstatements. + +> 💡 If anything goes wrong, you can run `docker logs ` to see some hopefully helpful error messages. + +#### Stopping + +To stop, use + +```sh +docker compose stop +``` + +#### Resetting the configuration + +Most values set in `.env` are written into the respective containers after you run `docker compose up` for the first time. + +If you want to reset the configuration while retaining your existing data: + +1. Make any needed changes to the values in the `.env` file copied from `template.env` above. NOTE: Do not change `DB_*` values unless you are also [re-creating the database](#removing-wikibase-suite-completely-with-all-its-data). +2. Delete your `LocalSettings.php` file from the `./config` directory. +3. Remove and re-create containers: + +```sh +docker compose down +docker compose up +``` + +### Advanced configuration +On first launch, WBS Deploy will create files in the `./config` directory alongside your `.env` file, the `docker-compose.yml` and `template.env`. This is your instance configuration. **You own and control those files.** Be sure to include them in your backups. + +#### `config/LocalSettings.php` + +This file is generated by the [MediaWiki installer script](https://www.mediawiki.org/wiki/Manual:Install.php) and supplemented by the Wikibase container's `entrypoint.sh` script on first launch. Once this file has been generated, you own and control it. This means that not only *can* you make changes to it, you may *need* to do so for [major version updates](https://www.mediawiki.org/wiki/Manual:Upgrading#Adapt_your_LocalSettings.php). + +If `config/LocalSettings.php` is missing, it triggers the Wikibase container to run the [MediaWiki installer script](https://www.mediawiki.org/wiki/Manual:Install.php). If you need to run the installer again, you can remove the generated `LocalSettings.php` file (but keep a backup just in case!) and restart your instance. + +#### `config/wikibase-php.ini` +This is Wikibase's `php.ini` override file, a good place for tuning PHP configuration values. It gets loaded by the Wikibase web server's PHP interpreter. + +#### docker-compose.yml +To further customize your instance, you can also make changes to `docker-compose.yml`. To ease updating to newer versions of WBS Deploy, consider putting your customizations into a new file called `docker-compose.override.yml`. If you do this, you'll need to start using the following commands to restart your instance: + +```sh +docker compose -f docker-compose.yml -f docker-compose.override.yml down +docker compose -f docker-compose.yml -f docker-compose.override.yml up --wait +``` + +This way, your changes are kept separate from the original WBS Deploy code. + +### Managing your data +Besides [your configuration](#configuring-your-wikibase-suite), it's your data that makes your instance unique. All instance data is stored in [Docker volumes](https://docs.docker.com/storage/volumes/). + + - `wikibase-image-data`: MediaWiki image and media file uploads + - `mysql-data`: MediaWiki/Wikibase MariaDB raw database + - `wdqs-data`: Wikidata Query Service raw database + - `elasticsearch-data`: Elasticsearch raw database + - `quickstatements-data`: generated Quickstatements OAuth binding for this MediaWiki instance + - `traefik-letsencrypt-data`: SSL certificates + +#### Back up your data +To back up your data, shut down the instance and dump the contents of all Docker volumes into `.tar.gz` files. + +```sh +docker compose down + +for v in \ + wbs-deploy_elasticsearch-data \ + wbs-deploy_mysql-data \ + wbs-deploy_quickstatements-data \ + wbs-deploy_traefik-letsencrypt-data \ + wbs-deploy_wdqs-data \ + wbs-deploy_wikibase-image-data \ + ; do + docker run --rm --volume $v:/backup debian:12-slim tar cz backup > $v.tar.gz +done +``` + +#### Restore from a backup + +To restore the volume backups, ensure your instance has been shut down by running `docker compose down` and populate the Docker volumes with data from your `.tar.gz` files. + +```sh +docker compose down + +for v in \ + wbs-deploy_elasticsearch-data \ + wbs-deploy_mysql-data \ + wbs-deploy_quickstatements-data \ + wbs-deploy_traefik-letsencrypt-data \ + wbs-deploy_wdqs-data \ + wbs-deploy_wikibase-image-data \ + ; do + docker volume rm $v 2> /dev/null + docker volume create $v + docker run -i --rm --volume $v:/backup debian:12-slim tar xz < $v.tar.gz +done +``` + +### Updating and versioning + +WBS uses [semantic versioning](https://semver.org/spec/v2.0.0.html). The WBS Deploy and all the WBS images have individual version numbers. + +WBS Deploy always references the latest minor and patch releases of the compatible WBS images' major versions using the images' major version tag. + +#### Example + +Let's say the `wikibase` image version 1.0.0 is the initial version released with WBS Deploy 3.0.0. In that case, the `wikibase` image carrying the `1.0.0` tag will also carry a `1` tag. When the `wikibase` image version is bumped to 1.1.0 for a feature release, a new image is released and tagged with `1.1.0`. The `1` tag will then be reused and now point to the newly released image 1.1.0. + +This way, WBS Deploy can always reference the latest compatible version by using the major version tag. Nothing needs to be updated in WBS Deploy itself. If the `wikibase` image version gets bumped to 2.0.0, that indicates a breaking change; in this case the new image would not receive the `1` tag. Instead, a new version of WBS Deploy would be released (in this case 4.0.0) and this one would use a new major version tag called `2` to reference the Wikibase image. + +WBS Deploy may also receive minor and patch updates, but, as noted above, they are not required to update related WBS images. + +#### Minor and patch updates for WBS images + +Because WBS Deploy always references the latest minor and patch releases of compatible WBS images, non-breaking changes (including security updates) can be pulled at any time. + +This is always safe to do. Simply run: + +```sh +docker compose down +docker compose pull +docker compose up +``` +> 💡 In order to automatically update images on every start, you can also use `docker compose up --pull always` to start your WBS Deploy stack. + +#### Minor and patch updates for WBS Deploy + +WBS Deploy major versions are tracked in dedicated branches such as `deploy-3`. Pulling from the major version branch you are currently on will only update minor and patch versions and will never trigger breaking changes. + +These updates are **always** considered safe. + +If you did not change `docker-compose.yml`, you can update simply by running `git pull`. + +```sh +git pull +``` +> 💡 If you have made changes to `docker-compose.yml`, commit them to a separate branch and merge them with upstream changes as you see fit. + +> 💡 Each major version of WBS Deploy always references exactly one major version of each of the WBS images. Thus, updating WBS Deploy minor and patch versions from a major version's git branch will never lead to breaking changes in WBS service images. + +#### Major upgrades + +Major version upgrades are performed by updating WBS Deploy's major version. This is done by changing your git checkout to the new major version branch. This may reference new major versions of WBS images or involve breaking changes. In turn, those may require additional steps as described below. + +WBS only supports updating from one major version to the next version in sequence. In order to upgrade from 1.x.x to 3.x.x, you must first upgrade from 1.x.x to 2.x.x and then to 3.x.x. + +##### Bring down your instance + +```sh +docker compose down +``` + +##### Back up your data and config + +[Create a backup](#backup-your-data) of your data. + +Back up your `./config` directory as well using: +``` +cp -r ./config ./config-$(date +%Y%M%d%H%M%S) +``` + +> 💡 If you made changes to `docker-compose.yml`, commit them to a separate branch and merge them as you see fit in the next step. + +##### Pull new version + +WBS Deploy major versions are tracked in separate branches called `deploy-MAJOR_VERSION`, such as `deploy-2` or `deploy-3`. Change your checkout to the new major version branch. + +```sh +git remote update +git checkout deploy-MAJOR_VERSION +git pull +``` + +> 💡 If you made changes to `docker-compose.yml`, merge them as you see fit. + +##### Apply any changes to .env + +Look for changes in the new `template.env` that you might want to apply to your `.env` file. + +##### Apply any migrations for your version + +
WBS Deploy 2.x.x to 3.x.x (MediaWiki 1.41 to MediaWiki 1.42)

+ +Read the [MediaWiki UPGRADE file](https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/core/+/refs/heads/REL1_42/UPGRADE). + +No Wikibase-specific migrations are necessary. + +

+ +
WBS Deploy 1.x.x to 2.x.x (MediaWiki 1.39 to MediaWiki 1.41)

+ +Read the [MediaWiki UPGRADE file](https://gerrit.wikimedia.org/r/plugins/gitiles/mediawiki/core/+/refs/heads/REL1_41/UPGRADE). + +No Wikibase-specific migrations are necessary. + +

+ +##### Bring your instance back up + +``` +docker compose up +``` + +#### Automatic updates + +At the moment, WBS Deploy does not support automatic updates. To automatically deploy minor and patch updates including security fixes to your WBS images, [restart your instance](#minor-and-patch-updates-for-wbs-service-containers) on a regular basis with a systemd timer, cron job, or similar. + +#### Downgrades + +Downgrades are not supported. In order to revert an update, restore your data from a backup made prior to the upgrade. + +### Removing Wikibase Suite Completely with all its Data + +‼️ **This will destroy all data! [Back up](#back-up-your-data) anything you wish to retain.** + +To reset the configuration and data, remove the Docker containers, Docker volumes and the generated `config/LocalSettings.php` file. + +```sh +docker compose down --volumes +rm config/LocalSettings.php +``` + +Removing the `traefik-letsencrypt-data` volume will request a new certificate from LetsEncrypt on the next launch of your instance. Certificate generation on LetsEncrypt is [rate-limited](https://letsencrypt.org/docs/rate-limits/); eventually you may be blocked from generating new certificates **for multiple days**. To avoid that outcome, change to the LetsEncrypt staging server by appending the following line to the `traefik` `command` stanza of your `docker-compose.yml` file: +```yml + --certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory +``` + +## WDQS Frontend + +To interact with the WDQS frontend, navigate to the URL defined as `WDQS_FRONTEND_PUBLIC_HOST` in the `.env` file. By default, this is set to `wdqs-frontend.example`. + +Alternatively, send `GET` requests with your SPARQL query to the WDQS frontend endpoint: +`https://wdqs-frontend.example.com/proxy/wdqs/bigdata/namespace/wdq/sparql?query={SPARQL}` + + +## FAQ + +### Can I host WBS Deploy locally? + +Yes, WBS Deploy can be hosted locally for testing purposes by using the example domain names `*.example` from `template.env` in your `.env` file. Configure those domains in your host machine's `/etc/hosts` file, so that your browser (on your host machine) resolves `*.example` to `127.0.0.1` and access the local WBS Deploy instance. + +However, due to OAuth requirements, QuickStatements may not function properly without publicly accessible domain names for both the `WIKIBASE_PUBLIC_HOST` and `QUICKSTATEMENTS_PUBLIC_HOST`. Also, running locally without publicly accessible addresses will prevent the generation of a valid SSL certificate; to accessing locally running services, you will need to allow the invalid certificate when loading the page for the first time. + +### Can I migrate from another Wikibase installation to WBS Deploy? + +It is possible to migrate an existing Wikibase installation to WBS Deploy. The general procedure is as follows: + + - [Back up your MediaWiki](https://www.mediawiki.org/wiki/Manual:Backing_up_a_wiki) + - [Install Wikibase Suite](#initial-setup) as described above + - Re-apply any [changes](#customizing-your-wikibase-suite-mediawiki) to `config/LocalSettings.php` + - Import your database dump + - Regenerate the WDQS database + - Regenerate the Elasticsearch database + +### My WDQS Updater keeps crashing, what can I do? + +Check out the known issue in the [WDQS README](../build/wdqs/README.md#Known-issues). You may find your solution there in the form of a workaround. + +### Do you recommend any VPS hosting providers? + +As of this writing, we can offer no specific recommendations for VPS providers to host Wikibase Suite. The suite has been tested successfully on various providers; as long as the [minimum technical requirements](#hardware) are met, it should run as expected. + +### Where can I get further help? + +If you have questions not listed above or need help, use this [bug report form](https://phabricator.wikimedia.org/maniphest/task/edit/form/129/) to start a conversation with the engineering team. diff --git a/deploy-lts/config/README.md b/deploy-lts/config/README.md new file mode 100644 index 000000000..83bb3a161 --- /dev/null +++ b/deploy-lts/config/README.md @@ -0,0 +1,5 @@ +# Wikibase Suite Deploy Configuration + +This directory will contain your configuration files after the first run of Wikibase Suite Deploy. Feel free to make adjustments as you see fit. Back up this folder regularly in order to restore your configuration later. + +Find information on how to run Wikibase Suite Deploy in the [top level README](../README.md). diff --git a/deploy-lts/config/wikibase-php.ini b/deploy-lts/config/wikibase-php.ini new file mode 100644 index 000000000..a6f5aac3c --- /dev/null +++ b/deploy-lts/config/wikibase-php.ini @@ -0,0 +1,15 @@ +; ############################################################################## +; Wikibase Suite PHP settings +; ############################################################################## +; Feel free to adjust those settings according your needs. +; Changes will be picked up automatically by the wikibase container on restart. + +; Set the PHP upload limits be the same as the MediaWiki default of 100M. +; Should match the wgMaxUploadSize setting in LocalSettings.php +file_uploads = On +upload_max_filesize = 100M +post_max_size = 100M + +; Set the PHP memory and execution limits +memory_limit = 100M +max_execution_time = 600 diff --git a/deploy-lts/docker-compose.yml b/deploy-lts/docker-compose.yml new file mode 100644 index 000000000..37ef7bdcf --- /dev/null +++ b/deploy-lts/docker-compose.yml @@ -0,0 +1,207 @@ +name: wbs-deploy + +services: + # -------------------------------------------------- + # A. CORE WIKIBASE SUITE SERVICES + # -------------------------------------------------- + + wikibase: + image: wikibase/wikibase-lts:1 + depends_on: + mysql: + condition: service_healthy + elasticsearch: + condition: service_healthy + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.wikibase.rule=Host(`${WIKIBASE_PUBLIC_HOST}`)" + volumes: + - ./config:/config + - wikibase-image-data:/var/www/html/images + - quickstatements-data:/quickstatements/data + environment: + MW_ADMIN_NAME: ${MW_ADMIN_NAME} + MW_ADMIN_PASS: ${MW_ADMIN_PASS} + MW_ADMIN_EMAIL: ${MW_ADMIN_EMAIL} + MW_WG_SERVER: https://${WIKIBASE_PUBLIC_HOST} + DB_SERVER: mysql:3306 + DB_USER: ${DB_USER} + DB_PASS: ${DB_PASS} + DB_NAME: ${DB_NAME} + ELASTICSEARCH_HOST: elasticsearch + QUICKSTATEMENTS_PUBLIC_URL: https://${QUICKSTATEMENTS_PUBLIC_HOST} + healthcheck: + test: curl --silent --fail localhost/wiki/Main_Page + interval: 10s + start_period: 5m + + wikibase-jobrunner: + image: wikibase/wikibase-lts:1 + command: /jobrunner-entrypoint.sh + depends_on: + wikibase: + condition: service_healthy + restart: unless-stopped + volumes_from: + - wikibase + + mysql: + image: mariadb:10.11 + restart: unless-stopped + volumes: + - mysql-data:/var/lib/mysql + environment: + MYSQL_DATABASE: ${DB_NAME} + MYSQL_USER: ${DB_USER} + MYSQL_PASSWORD: ${DB_PASS} + MYSQL_RANDOM_ROOT_PASSWORD: yes + healthcheck: + test: healthcheck.sh --connect --innodb_initialized + start_period: 1m + interval: 20s + timeout: 5s + + # -------------------------------------------------- + # B. EXTRA WIKIBASE SUITE SERVICES + # -------------------------------------------------- + + # To disable Elasticsearch and use default MediaWiki search functionality remove + # the elasticsearch service, and the MW_ELASTIC_* vars from wikibase_variables + # at the top of this file. + elasticsearch: + image: wikibase/elasticsearch:1 + restart: unless-stopped + volumes: + - elasticsearch-data:/usr/share/elasticsearch/data + environment: + discovery.type: single-node + ES_JAVA_OPTS: -Xms512m -Xmx512m -Dlog4j2.formatMsgNoLookups=true + healthcheck: + test: curl --silent --fail localhost:9200 + interval: 10s + start_period: 2m + + wdqs: + image: wikibase/wdqs:2 + command: /runBlazegraph.sh + depends_on: + wikibase: + condition: service_healthy + restart: unless-stopped + # Set number of files ulimit high enough, otherwise blazegraph will abort with: + # library initialization failed - unable to allocate file descriptor table - out of memory + # Appeared on Docker 24.0.5, containerd 1.7.9, Linux 6.6.6, NixOS 23.11 + ulimits: + nofile: + soft: 32768 + hard: 32768 + volumes: + - wdqs-data:/wdqs/data + healthcheck: + test: curl --silent --fail localhost:9999/bigdata/namespace/wdq/sparql + interval: 10s + start_period: 2m + + wdqs-updater: + image: wikibase/wdqs:2 + command: /runUpdate.sh + depends_on: + wdqs: + condition: service_healthy + restart: unless-stopped + # Set number of files ulimit high enough, otherwise blazegraph will abort with: + # library initialization failed - unable to allocate file descriptor table - out of memory + # Appeared on Docker 24.0.5, containerd 1.7.9, Linux 6.6.6, NixOS 23.11 + ulimits: + nofile: + soft: 32768 + hard: 32768 + environment: + WIKIBASE_CONCEPT_URI: https://${WIKIBASE_PUBLIC_HOST} + + wdqs-proxy: + image: wikibase/wdqs-proxy:1 + depends_on: + wdqs: + condition: service_healthy + restart: unless-stopped + + wdqs-frontend: + image: wikibase/wdqs-frontend:1 + depends_on: + - wdqs-proxy + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.wdqs-frontend.rule=Host(`${WDQS_FRONTEND_PUBLIC_HOST}`)" + environment: + WDQS_HOST: wdqs-proxy + healthcheck: + test: curl --silent --fail localhost + interval: 10s + start_period: 2m + + quickstatements: + image: wikibase/quickstatements:1 + depends_on: + wikibase: + condition: service_healthy + restart: unless-stopped + volumes: + - quickstatements-data:/quickstatements/data + labels: + - "traefik.enable=true" + - "traefik.http.routers.quickstatements.rule=Host(`${QUICKSTATEMENTS_PUBLIC_HOST}`)" + environment: + QUICKSTATEMENTS_PUBLIC_URL: https://${QUICKSTATEMENTS_PUBLIC_HOST} + WIKIBASE_PUBLIC_URL: https://${WIKIBASE_PUBLIC_HOST} + healthcheck: + test: curl --silent --fail localhost + interval: 10s + start_period: 2m + + # -------------------------------------------------- + # C. REVERSE PROXY AND SSL SERVICES + # -------------------------------------------------- + + traefik: + image: traefik:3.1 + command: + # Basic setup + - "--providers.docker=true" + - "--providers.docker.exposedbydefault=false" + # http endpoint + - "--entrypoints.web.address=:80" + # https endpoint + - "--entrypoints.websecure.address=:443" + - "--entrypoints.websecure.asdefault" + - "--entrypoints.websecure.http.tls.certresolver=letsencrypt" + # http to https redirect + - "--entrypoints.web.http.redirections.entryPoint.to=websecure" + - "--entrypoints.web.http.redirections.entryPoint.scheme=https" + - "--entrypoints.web.http.redirections.entrypoint.permanent=true" + # ACME SSL certificate generation + - "--certificatesresolvers.letsencrypt.acme.httpchallenge=true" + - "--certificatesresolvers.letsencrypt.acme.httpchallenge.entrypoint=web" + - "--certificatesresolvers.letsencrypt.acme.email=${MW_ADMIN_EMAIL}" + - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json" + # Uncomment this line to only test ssl generation first, makes sure you don't run into letsencrypt rate limits + # - "--certificatesresolvers.letsencrypt.acme.caserver=https://acme-staging-v02.api.letsencrypt.org/directory" + ports: + - 80:80 + - 443:443 + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - traefik-letsencrypt-data:/letsencrypt + +volumes: + # A. CORE WIKIBASE SUITE SERVICES DATA + wikibase-image-data: + mysql-data: + # B. EXTRA WIKIBASE SUITE SERVICES DATA + wdqs-data: + elasticsearch-data: + quickstatements-data: + # C. REVERSE PROXY AND SSL SERVICES DATA + traefik-letsencrypt-data: diff --git a/deploy-lts/package.json b/deploy-lts/package.json new file mode 100644 index 000000000..0ee77cbc1 --- /dev/null +++ b/deploy-lts/package.json @@ -0,0 +1,12 @@ +{ + "name": "deploy-lts", + "version": "1.0.0", + "nx": { + "targets": { + "lint": {}, + "nx-release-publish": { + "command": "echo \"⚠️ Make sure to updated related documentation to point to new Deploy release (nothing changed)\"" + } + } + } +} diff --git a/deploy-lts/template.env b/deploy-lts/template.env new file mode 100644 index 000000000..216749c26 --- /dev/null +++ b/deploy-lts/template.env @@ -0,0 +1,37 @@ +# ############################################################################## +# Wikibase Suite Deploy - initial configuration +# +# This file is a template for the .env file used by the Wikibase Suite Deploy +# docker-compose setup. +# +# WARNING: Do not add comments on the same line as env vars: in some +# environments they will be included in the var! +# ############################################################################## + +# Public hostname configuration. +# These domain names for your Wikibase Suite services should be configured on +# your DNS server to point to the public IP address of the server being +# deployed to. Note that you need three distinct names, e.g. three different +# subdomains. Otherwise the reverse proxy cannot route properly. +WIKIBASE_PUBLIC_HOST=wikibase.example +WDQS_FRONTEND_PUBLIC_HOST=wdqs-frontend.example +QUICKSTATEMENTS_PUBLIC_HOST=quickstatements.example + +# MediaWiki / Wikibase user configuration. +# Password must be at least 10 characters, different from your username and not +# appear in the list of commonly used passwords this project uses. +MW_ADMIN_NAME=admin +MW_ADMIN_EMAIL=admin@wikibase.example +MW_ADMIN_PASS=change-this-password + +# MediaWiki / Wikibase database configuration. +# Those settings are used by the MariaDB container when creating a new +# database and by MediaWiki when generating a new LocalSettings.php. They will not +# be set on an existing database, nor will MediaWiki update those settings in +# your LocalSettings.php. To change those settings, either adjust them manually +# in MariaDB and your LocalSettings.php, or delete your MariaDB volume +# "mysql-data" and your LocalSettings.php from the ./config directory and +# restart. +DB_NAME=my_wiki +DB_USER=sqluser +DB_PASS=change-this-password diff --git a/nx.json b/nx.json index d86ca0ec0..687232fbe 100644 --- a/nx.json +++ b/nx.json @@ -27,7 +27,7 @@ "projects": [ "build/*" ] }, "deploy": { - "projects": [ "deploy" ] + "projects": [ "deploy", "deploy-lts" ] } }, "projectsRelationship": "independent", diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index f13886d89..d31b9c298 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -2,3 +2,4 @@ packages: - "build/*" - "test/**" - "deploy/**" + - "deploy-lts/**"