From 897d9f30a0e534971f5b91d7cf40b46cd69e60c8 Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 25 Nov 2024 17:23:25 -0500 Subject: [PATCH] Sam/oriole17 (#1298) * feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * pg 15 and 16 packer/ansible/ghactions (#1268) * fix: reformat ec2 cleanup commands (#1267) Co-authored-by: Sam Rose * feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * feat: realease 15 and 16 to staging * chore: update versions * chore: make yq available * chore: run yq from nix * chore: more setup for staging AMI * fix: yq usage * chore: shell vars * fix: When --init none is used, only users who can elevate to sudo privileges can run Nix * fix: no -i * fix: quote correction * fix: newline extra quotes * fix: no need for pg major version on packer * fix: postgresql_major * fix: ql * fix: no ansible args in stage to invocation * fix: unique val * fix: adjustments to build scripts * chore: env var handling * fix: bump to build * chore: set up more required vars * chore: bump var * feat: pg 16 debug symbols * feat: matrix pg versions build on testinfra * feat: matrix on Test Database * chore: running nix in the right context * feat: just use existing Dockerfile + pg version * chore: refer to var * fix: read name without including quotes * chore: try format function * fix: strip quotes from version number * chore: env var * fix: pg client * fix * fix: try to use psql from our own corresponding pkg * fix: try psql from ppa * fix: dbmate per pg version * build dbmate and then install client * fix: account for architecture * chore: limit changes detection migrations/schema.sql * missing docker compose call * ore: drop tests while investigating * test: try on pg15 only * chore: schema needs update * chore: now run on all versions in matrix * test: trying a version of schema per major pg version as there are type diffs --------- Co-authored-by: Sam Rose Co-authored-by: Oliver Rice * chore: rm dead code * chore: resolve versions from ansible/vars.yml instead of packer file create a matrix build for 15 16 (or how ever many versions there are) * feat: WIP orioledb 16 * feat: intro into CI building psql16 bundle oriole version * test: working through build compat for oriole * working orioledb build * fix: update value * chore: cut staging only release for oriole16 * narrow down to just oriole staging ami here * fix: make sure name lines up with gh action convention * fix: get correct naming convention in here too for oriole bundle * chore: bump to cut release * fix: make sure src and debug builds available for oriole too * chore: bump to staging release * fix: when oriole16 rm timescale from supautils conf * fix: also remove from postgresql conf * fix: find all instances and remove in this condition for tiemscaledb * fix: maybe broke pgsodium with too many spaces * fix: create a symlink for missing path * fix: try a more universal way to deactivate pgsodium at this stage * chore: when oriole16 rm timescale plv8 postgis pgvector pgrouting * fix: more handling of oriole16 differences * chore more oriole16 handling * fix: refine sed regex * fix: also cond rm actual files * fix: correct dir * chore: bump to release * feat: parse and handle oropledb-16 in start-server * feat: start-client for oriole16 * wip for orioledb push to build on linux * fix: running server * chore: consolidate nix code that handles building of postgres versions * fix: update name for bundle * chore: small cleanup * chore: bump version release * chore: cleanup names * chore: correct names * fix: var name * fix: one more var name * chore: bump to release * fix: reformatting metadata for clikchouse dep as git was not able to fetch * feat: wip refactor to pg 17 for orioledb * feat: working orioledb-17 * fix: increase role duration to avoid expiration * fix: correct version * fix: clickhouse needs git in build inputs * fix: install and then remove git for clickhouse dep * fix: correct rules for version * fix: db_user_namespace was deprecated starting in pg 16.4 and higher * apply fix for wrappers build, deactivate ext in unit test * chore: deactivate more tests due to deactivated extensions * fix: also remove from sql file * fix: regex * fix: also remove ref here * fix: graphql_public schema too * chore: correct dir * chore: staging release * feat: re-introduce wal2json rum and pgvector * fix: also restore pgvector in start-server * chore: bump staging release * feat: oriole specific default settings * feat: use icu for locales only if building orioledb * fix: need var in stage 1 too * fix: settings must be in db init * fix: ansible args * fix: PostgreSQL uses ICU for all locale-related operations, so we don't need to specify the collation and character type settings separately. * chore: add a migration for orioledb activation by default * feat: turn on oriole if oriole ext exists * test: conditional on test for orioledb * fix: salt changed repo and method for adding apt package * fix: udpate salt minion setup * fix: silent skip if oriole not in the install * test: check if oriole is available and if so then enable * chore: build and cache src and debug pkgs as well * fix: tmp supress build of src and debug * chore: tmp disable src and debug * fix: activate oriole first in the oriole context * fix: pgroonga build on macos * fix: restore flake check for now * fixing flake check for darwin + passing check until pg_net * fix: start-server macos * fix: src yq from apt * fix: all instances yq * fix: jq invoke * fix: nix profile install * fix: nix run * fix: sudo nix run * chore: cut staging release * chore: rm debugging * fix: still need CURRENT_SYSTEM * test: re-introduce flake checks for 15 and 16 * chore: bump versions of patched pg for oriole to 17_4 and ext to beta6 * chore: bump var to release ami * chore: break down into functions * fix: skip x86 darwin for oriole nix builds * chore: cleanup repo for final review, rebase and merge * fix: try to properly init db in non oriole context * chore: restore installation of debug and src for all versions` * chore: newline * chore: cleaning up and restoring missing code * chore: cleanup * chore: fix gh action conditonal for oriole * fix: filter orioledb-17 from test on read --------- Co-authored-by: Oliver Rice --- .github/workflows/test.yml | 21 +-- .github/workflows/testinfra-nix.yml | 3 +- .gitignore | 2 +- ansible/files/postgres_prestart.sh.j2 | 44 +++++- ansible/tasks/setup-postgres.yml | 60 +++++--- ansible/tasks/stage2-setup-postgres.yml | 77 ++++++++-- ansible/tasks/test-image.yml | 71 ++++++++- ansible/vars.yml | 9 +- docker/nix/build_nix.sh | 30 ++-- .../files/unit-tests/unit-test-01.sql | 63 ++++++-- .../scripts/surrogate-bootstrap-nix.sh | 5 +- flake.nix | 144 ++++++++---------- .../20241031003909_create_orioledb.sql | 11 ++ migrations/tests/test.sql | 10 ++ nix/ext/orioledb.nix | 10 +- nix/ext/pgaudit.nix | 4 + nix/ext/pgroonga.nix | 24 ++- nix/ext/rum.nix | 4 +- nix/ext/wal2json.nix | 4 +- nix/ext/wrappers/default.nix | 36 +++++ nix/overlays/psql_16-oriole.nix | 12 +- nix/postgresql/default.nix | 5 +- nix/postgresql/generic.nix | 29 ++-- nix/postgresql/orioledb-16.nix | 4 + nix/postgresql/orioledb-17.nix | 4 + nix/tools/run-client.sh.in | 6 +- nix/tools/run-server.sh.in | 35 ++++- 27 files changed, 525 insertions(+), 202 deletions(-) create mode 100644 migrations/db/migrations/20241031003909_create_orioledb.sql create mode 100644 nix/postgresql/orioledb-16.nix create mode 100644 nix/postgresql/orioledb-17.nix diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d22b16f16..1b60cfcca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,10 @@ name: Test Database - on: push: branches: - develop pull_request: workflow_dispatch: - jobs: prepare: runs-on: ubuntu-latest @@ -21,13 +19,11 @@ jobs: extra-conf: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[] | select(. != "orioledb-17")' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT - build: needs: prepare strategy: @@ -45,34 +41,25 @@ jobs: POSTGRES_PASSWORD: password steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main with: extra-conf: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Strip quotes from pg major and set env var run: | stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl run: | PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - id: settings - # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - name: Generate args id: args run: | @@ -80,14 +67,12 @@ jobs: echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - #TODO PR Convert to develop branch flakeurl - name: verify schema.sql is committed run: | GIT_SHA=${{github.sha}} - nix run github:supabase/postgres/${GIT_SHA}#dbmate-tool -- --version ${{ env.PGMAJOR }} + nix run github:supabase/postgres/develop#dbmate-tool -- --version ${{ env.PGMAJOR }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql exit 1 - fi + fi \ No newline at end of file diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 4a51c159c..14db4a7f2 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -18,7 +18,7 @@ jobs: - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT test-ami-nix: @@ -61,6 +61,7 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + - name: Generate common-nix.vars.pkr.hcl run: | diff --git a/.gitignore b/.gitignore index ae375b82a..45464e3f0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ result* .idea/ .vscode/ -db \ No newline at end of file +db diff --git a/ansible/files/postgres_prestart.sh.j2 b/ansible/files/postgres_prestart.sh.j2 index ae5aa1a7d..3ffe54c85 100644 --- a/ansible/files/postgres_prestart.sh.j2 +++ b/ansible/files/postgres_prestart.sh.j2 @@ -1,9 +1,49 @@ #!/bin/bash +check_orioledb_enabled() { + local pg_conf="/etc/postgresql/postgresql.conf" + if [ ! -f "$pg_conf" ]; then + return 0 + fi + grep "^shared_preload_libraries" "$pg_conf" | grep -c "orioledb" || return 0 +} + +get_shared_buffers() { + local opt_conf="/etc/postgresql-custom/generated-optimizations.conf" + if [ ! -f "$opt_conf" ]; then + return 0 + fi + grep "^shared_buffers = " "$opt_conf" | cut -d "=" -f2 | tr -d ' ' || return 0 +} + +update_orioledb_buffers() { + local pg_conf="/etc/postgresql/postgresql.conf" + local value="$1" + if grep -q "^orioledb.main_buffers = " "$pg_conf"; then + sed -i "s/^orioledb.main_buffers = .*/orioledb.main_buffers = $value/" "$pg_conf" + else + echo "orioledb.main_buffers = $value" >> "$pg_conf" + fi +} + +main() { + local has_orioledb=$(check_orioledb_enabled) + if [ "$has_orioledb" -lt 1 ]; then + return 0 + fi + local shared_buffers_value=$(get_shared_buffers) + if [ ! -z "$shared_buffers_value" ]; then + update_orioledb_buffers "$shared_buffers_value" + fi +} + +# Initial locale setup if [ $(cat /etc/locale.gen | grep -c en_US.UTF-8) -eq 0 ]; then - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen fi if [ $(locale -a | grep -c en_US.utf8) -eq 0 ]; then - locale-gen + locale-gen fi + +main diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 0038e3afc..188a364b8 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -204,23 +204,49 @@ ansible_command_timeout: 60 when: debpkg_mode -- name: Initialize the database stage2_nix - become: yes - become_user: postgres - shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" - args: - executable: /bin/bash - environment: - LANG: en_US.UTF-8 - LANGUAGE: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - LC_CTYPE: en_US.UTF-8 - LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - vars: - ansible_command_timeout: 60 - # Circumvents the following error: - # "Timeout (12s) waiting for privilege escalation prompt" - when: stage2_nix +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary + block: + - name: Check if psql_version is psql_orioledb + set_fact: + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" + + - name: Initialize the database stage2_nix (non-orioledb) + become: yes + become_user: postgres + shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + vars: + ansible_command_timeout: 60 + when: stage2_nix and not is_psql_oriole + + - name: Initialize the database stage2_nix (orioledb) + become: yes + become_user: postgres + shell: > + source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data + --allow-group-access + --username=supabase_admin + --locale-provider=icu + --encoding=UTF-8 + --icu-locale=en_US.UTF-8 + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + vars: + ansible_command_timeout: 60 + when: stage2_nix and is_psql_oriole - name: copy PG systemd unit template: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 548f3ed94..c3ce46941 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -4,11 +4,56 @@ # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary + block: + - name: Check if psql_version is psql_orioledb-16 + set_fact: + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" + + - name: Remove specified extensions from postgresql.conf if oriole-16 build + ansible.builtin.command: + cmd: > + sed -i 's/ timescaledb,//g' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Remove specified extensions from supautils.conf if oriole-16 build + ansible.builtin.command: + cmd: > + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' + /etc/postgresql-custom/supautils.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Remove db_user_namespace from postgresql.conf if oriole-xx build + ansible.builtin.command: + cmd: > + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Append orioledb to shared_preload_libraries append within closing quote + ansible.builtin.command: + cmd: > + sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Add default_table_access_method setting + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "default_table_access_method = 'orioledb'" + state: present + when: is_psql_oriole and stage2_nix + become: yes + - name: Install Postgres from nix binary cache become: yes shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" -#TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix - name: Install pg_prove from nix binary cache @@ -199,13 +244,19 @@ recurse: yes when: stage2_nix -- name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir - shell: > - sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ - sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" - become: yes - when: stage2_nix +- name: Check psql_version and run postgis linking if not oriole-xx + block: + - name: Check if psql_version is psql_orioledb-17 + set_fact: + is_psql_oriole: "{{ psql_version == 'psql_orioledb-17' }}" + + - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir + shell: > + sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ + sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" + become: yes + when: stage2_nix and not is_psql_oriole - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: @@ -240,8 +291,16 @@ line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh' when: stage2_nix +- name: Create symbolic link for pgsodium_getkey script + file: + src: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" + dest: "/usr/lib/postgresql/share/postgresql/extension/pgsodium_getkey" + state: link + become: yes + when: stage2_nix + - name: Append GRN_PLUGINS_DIR to /etc/environment.d/postgresql.env ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env line: 'GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins' - become: yes \ No newline at end of file + become: yes diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index d6e8223f7..d8e951a3e 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -4,11 +4,22 @@ - libtap-parser-sourcehandler-pgtap-perl when: debpkg_mode +# - name: Temporarily disable PG Sodium references in config +# become: yes +# become_user: postgres +# shell: +# cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf +# when: debpkg_mode or stage2_nix + - name: Temporarily disable PG Sodium references in config become: yes become_user: postgres shell: - cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + cmd: > + sed -i.bak + -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' + -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/' + /etc/postgresql/postgresql.conf when: debpkg_mode or stage2_nix - name: Start Postgres Database to load all extensions. @@ -46,6 +57,64 @@ LOCALE_ARCHIVE: /usr/lib/locale/locale-archive when: stage2_nix + +- name: Check psql_version and modify migrations if oriole-xx + block: + - name: Check if psql_version is psql_orioledb-xx + set_fact: + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" + + - name: Remove specified extensions from SQL file + ansible.builtin.command: + cmd: > + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql + when: is_psql_oriole + become: yes + + - name: Remove specified extension files from extensions directory + ansible.builtin.find: + paths: /tmp/migrations/tests/extensions + patterns: + - '*timescaledb*.sql' + - '*postgis*.sql' + - '*pgrouting*.sql' + - '*plv8*.sql' + - '*pg_jsonschema*.sql' + - '*pg_graphql*.sql' + register: files_to_remove + when: is_psql_oriole + + - name: Delete matched extension files + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ files_to_remove.files }}" + when: is_psql_oriole + become: yes + + - name: Remove specified extensions from SQL file + ansible.builtin.command: + cmd: > + sed -i "/'pg_graphql',/d" /tmp/unit-tests/unit-test-01.sql + when: is_psql_oriole + become: yes + + - name: Remove graphql schema test + lineinfile: + path: /tmp/migrations/tests/database/exists.sql + regexp: "^SELECT has_schema\\('graphql'\\);$" + state: absent + become: yes + when: is_psql_oriole + + - name: Remove graphql schema test + lineinfile: + path: /tmp/migrations/tests/database/exists.sql + regexp: "^SELECT has_schema\\('graphql_public'\\);$" + state: absent + become: yes + when: is_psql_oriole + - name: Run Unit tests (with filename unit-test-*) on Postgres Database shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql register: retval diff --git a/ansible/vars.yml b/ansible/vars.yml index 1209c29c7..0bcc68ab3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,17 +2,16 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -# postgresql_major: "15" -# postgresql_release: "15.1" -# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c postgres_major: - "15" - "16" + - "orioledb-17" # Full version strings for each major version postgres_release: - postgres15: "15.8.1.012" - postgres16: "16.3.1.018" + postgresorioledb-17: "orioledb-17.0.1.000" + postgres15: "15.8.1.005" + postgres16: "16.3.1.011" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 5a7896a46..fc9883797 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,15 +5,23 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi -if [ $(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') == "x86_64-darwin" ]; then - nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link - nix build .#psql_15/bin -o psql_15 - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 -else - nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link - nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link - nix build .#psql_15/bin -o psql_15 - nix build .#psql_16/bin -o psql_16 - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 - nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 + +SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') + +nix build .#checks.$SYSTEM.psql_15 -L --no-link +nix build .#checks.$SYSTEM.psql_16 -L --no-link + +nix build .#psql_15/bin -o psql_15 +nix build .#psql_16/bin -o psql_16 + +# Skip orioledb-17 on x86_64-darwin +if [ "$SYSTEM" != "x86_64-darwin" ]; then + nix build .#psql_orioledb-17/bin -o psql_orioledb_17 +fi + +# Copy to S3 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 +if [ "$SYSTEM" != "x86_64-darwin" ]; then + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 fi diff --git a/ebssurrogate/files/unit-tests/unit-test-01.sql b/ebssurrogate/files/unit-tests/unit-test-01.sql index 0feb70e8b..f3d47459f 100644 --- a/ebssurrogate/files/unit-tests/unit-test-01.sql +++ b/ebssurrogate/files/unit-tests/unit-test-01.sql @@ -1,33 +1,64 @@ BEGIN; CREATE EXTENSION IF NOT EXISTS pgtap; + +DO $$ +DECLARE + extension_array text[]; + orioledb_available boolean; +BEGIN + -- Check if orioledb is available + SELECT EXISTS ( + SELECT 1 FROM pg_available_extensions WHERE name = 'orioledb' + ) INTO orioledb_available; + + -- If available, create it and add to the expected extensions list + IF orioledb_available THEN + CREATE EXTENSION IF NOT EXISTS orioledb; + extension_array := ARRAY[ + 'plpgsql', + 'pg_stat_statements', + 'pgsodium', + 'pgtap', + 'pg_graphql', + 'pgcrypto', + 'pgjwt', + 'uuid-ossp', + 'supabase_vault', + 'orioledb' + ]; + ELSE + extension_array := ARRAY[ + 'plpgsql', + 'pg_stat_statements', + 'pgsodium', + 'pgtap', + 'pg_graphql', + 'pgcrypto', + 'pgjwt', + 'uuid-ossp', + 'supabase_vault' + ]; + END IF; + + -- Set the array as a temporary variable to use in the test + PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false); +END $$; + SELECT plan(8); --- Check installed extensions SELECT extensions_are( - ARRAY[ - 'plpgsql', - 'pg_stat_statements', - 'pgsodium', - 'pgtap', - 'pg_graphql', - 'pgcrypto', - 'pgjwt', - 'uuid-ossp', - 'supabase_vault' - ] + string_to_array(current_setting('myapp.extensions'), ',')::text[] ); --- Check schemas exists SELECT has_schema('pg_toast'); SELECT has_schema('pg_catalog'); SELECT has_schema('information_schema'); SELECT has_schema('public'); --- Check that service_role can execute certain pgsodium functions SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']); -SELECT * from finish(); -ROLLBACK; +SELECT * FROM finish(); +ROLLBACK; \ No newline at end of file diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 3b0bb93c8..b204b326a 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -219,7 +219,10 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' $ARGS + ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \ + --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \ + --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \ + $ARGS } function update_systemd_services { diff --git a/flake.nix b/flake.nix index 71b87d580..d1892b117 100644 --- a/flake.nix +++ b/flake.nix @@ -26,26 +26,6 @@ pgsqlSuperuser = "supabase_admin"; nix2img = nix2container.packages.${system}.nix2container; - # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which - # we can use to build our own images; it is the common name to refer to - # a copy of nixpkgs which contains all its packages. - # it also serves as a base for importing the orioldb/postgres overlay to - #build the orioledb postgres patched version of postgresql16 - oriole_pkgs = import nixpkgs { - config = { allowUnfree = true; }; - inherit system; - overlays = [ - # NOTE (aseipp): add any needed overlays here. in theory we could - # pull them from the overlays/ directory automatically, but we don't - # want to have an arbitrary order, since it might matter. being - # explicit is better. - (import ./nix/overlays/cargo-pgrx.nix) - (import ./nix/overlays/psql_16-oriole.nix) - - ]; - }; - #This variable works the same as 'oriole_pkgs' but builds using the upstream - #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { config = { allowUnfree = true; @@ -55,7 +35,7 @@ }; inherit system; overlays = [ - # NOTE add any needed overlays here. in theory we could + # NOTE: add any needed overlays here. in theory we could # pull them from the overlays/ directory automatically, but we don't # want to have an arbitrary order, since it might matter. being # explicit is better. @@ -92,11 +72,7 @@ }) (final: prev: { postgresql = final.callPackage ./nix/postgresql/default.nix { - inherit (final) lib; - inherit (final) stdenv; - inherit (final) fetchurl; - inherit (final) makeWrapper; - inherit (final) callPackage; + inherit (final) lib stdenv fetchurl makeWrapper callPackage buildEnv newScope; }; }) ]; @@ -165,11 +141,19 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option - orioledbExtension = ourExtensions ++ [ ./nix/ext/orioledb.nix ]; + #we're not using timescaledb in the orioledb version of supabase extensions + orioleFilteredExtensions = builtins.filter ( + x: + x != ./nix/ext/timescaledb.nix && + x != ./nix/ext/plv8.nix && + x != ./nix/ext/postgis.nix && + x != ./nix/ext/pgrouting.nix && + x != ./nix/ext/pg_jsonschema.nix && + x != ./nix/ext/pg_graphql.nix + ) ourExtensions; + + orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - #this var is a convenience setting to import the orioledb patched version of postgresql - postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; - #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way @@ -200,7 +184,7 @@ }; extensions = ourExts; - # NOTE (aseipp): this field can be used to do cache busting (e.g. + # NOTE this field can be used to do cache busting (e.g. # force a rebuild of the psql packages) but also to helpfully inform # tools what version of the schema is being used, for forwards and # backwards compatibility @@ -208,28 +192,13 @@ }; }; - makeOurOrioleDbPostgresPkgs = version: patchedPostgres: - let postgresql = patchedPostgres; - in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; - makeOurPostgresPkgs = version: let postgresql = getPostgresqlPackage version; - extensions = if version == "15" - then ourExtensions ++ [ - ./nix/ext/timescaledb-2.9.1.nix - ] + extensionsToUse = if (builtins.elem version ["orioledb-16" "orioledb-17"]) + then orioledbExtensions else ourExtensions; - in - map (path: pkgs.callPackage path { inherit postgresql; }) extensions; - # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. - makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: - (builtins.listToAttrs (map - (drv: - { name = drv.pname; value = drv; } - ) - (makeOurOrioleDbPostgresPkgs version patchedPostgres))) - // { recurseForDerivations = true; }; + in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: @@ -269,27 +238,6 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; - makeOrioleDbPostgresBin = version: patchedPostgres: - let - postgresql = patchedPostgres; - upstreamExts = map - (ext: { - name = postgresql.pkgs."${ext}".pname; - version = postgresql.pkgs."${ext}".version; - }) - orioledbPsqlExtensions; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); - - pgbin = postgresql.withPackages (ps: - (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) - ); - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - }; - - # Create an attribute set, containing all the relevant packages for a # PostgreSQL install, wrapped up with a bow on top. There are three # packages: @@ -304,11 +252,6 @@ exts = makeOurPostgresPkgsSet version; recurseForDerivations = true; }; - makeOrioleDbPostgres = version: patchedPostgres: rec { - bin = makeOrioleDbPostgresBin version patchedPostgres; - exts = makeOurOrioleDbPostgresPkgsSet version patchedPostgres; - recurseForDerivations = true; - }; # The base set of packages that we export from this Nix Flake, that can # be used with 'nix build'. Don't use the names listed below; check the @@ -326,7 +269,8 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + #psql_orioledb-16 = makePostgres "orioledb-16" ; + psql_orioledb-17 = makePostgres "orioledb-17" ; }; # Find the active PostgreSQL version @@ -342,6 +286,8 @@ }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; + #postgresql_orioledb-16 = getPostgresqlPackage "orioledb-16"; + postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; in postgresVersions //{ supabase-groonga = supabase-groonga; @@ -350,12 +296,14 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + #psql_orioledb-16 = postgresVersions.psql_orioledb-16; + psql_orioledb-17 = postgresVersions.psql_orioledb-17; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16; + inherit postgresql_15 postgresql_16 postgresql_orioledb-17; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; + postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -400,6 +348,28 @@ platforms = platforms.all; }; }; + postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-17-src"; + version = postgresql_orioledb-17.version; + + src = postgresql_orioledb-17.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; @@ -459,6 +429,7 @@ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -468,7 +439,8 @@ --subst-var-by 'LOCALES' '${localeArchive}' \ --subst-var-by 'EXTENSION_CUSTOM_SCRIPTS_DIR' "$out/extension-custom-scripts" \ --subst-var-by 'MECAB_LIB' '${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so' \ - --subst-var-by 'GROONGA_DIR' '${supabase-groonga}' + --subst-var-by 'GROONGA_DIR' '${supabase-groonga}' \ + --subst-var-by 'CURRENT_SYSTEM' '${system}' chmod +x $out/bin/start-postgres-server ''; @@ -488,6 +460,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ @@ -583,19 +556,24 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; pg_regress = basePackages.pg_regress; + tmpDirCmd = if pkgs.stdenv.isDarwin then + ''mkdir -p /tmp/postgres-check.$$ && echo "/tmp/postgres-check.$$"'' + else + "mktemp -d"; in pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove pg_regress procps supabase-groonga ]; } '' - TMPDIR=$(mktemp -d) + TMPDIR=$(${tmpDirCmd}) if [ $? -ne 0 ]; then echo "Failed to create temp directory" >&2 exit 1 fi + chmod -R 755 "$TMPDIR" # Ensure the temporary directory is removed on exit - trap 'rm -rf "$TMPDIR"' EXIT + #trap 'rm -rf "$TMPDIR"' EXIT export PGDATA="$TMPDIR/pgdata" export PGSODIUM_DIR="$TMPDIR/pgsodium" @@ -615,6 +593,8 @@ echo "listen_addresses = '*'" >> $PGDATA/postgresql.conf echo "port = 5432" >> $PGDATA/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf + # Add system-specific configuration for aarch64-darwin + #postgres -D "$PGDATA" -k "$TMPDIR" -h localhost -p 5432 >$TMPDIR/logfile/postgresql.log 2>&1 & pg_ctl -D "$PGDATA" -l $TMPDIR/logfile/postgresql.log -o "-k $TMPDIR -p 5432" start for i in {1..60}; do @@ -655,7 +635,7 @@ pg_ctl -D "$PGDATA" stop mv $TMPDIR/logfile/postgresql.log $out echo ${pgpkg} - ''; + ''; in rec { # The list of all packages that can be built with 'nix build'. The list diff --git a/migrations/db/migrations/20241031003909_create_orioledb.sql b/migrations/db/migrations/20241031003909_create_orioledb.sql new file mode 100644 index 000000000..dbfe5a63e --- /dev/null +++ b/migrations/db/migrations/20241031003909_create_orioledb.sql @@ -0,0 +1,11 @@ +do $$ +begin + if exists (select 1 from pg_available_extensions where name = 'orioledb') then + if not exists (select 1 from pg_extension where extname = 'orioledb') then + create extension if not exists orioledb; + end if; + end if; +end $$; + +-- migrate:down +drop extension if exists orioledb; diff --git a/migrations/tests/test.sql b/migrations/tests/test.sql index 7afa40645..9682b4a2b 100644 --- a/migrations/tests/test.sql +++ b/migrations/tests/test.sql @@ -1,3 +1,13 @@ +-- Check and create OrioleDB if available +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'orioledb') THEN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'orioledb') THEN + CREATE EXTENSION orioledb; + END IF; + END IF; +END $$; + -- Create all extensions \ir extensions/test.sql diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 4d8c51bbd..9775226e1 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,17 +6,17 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "main"; - sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY="; + rev = "beta6"; + sha256 = "sha256-7nmgQDVcNuaGOM6V9xrer2jqfxhJhXZBXRZoiPEoisE="; }; - version = "patches16_23"; + version = "beta6"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=4"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index f6c5d8bb0..84fbf440d 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -2,6 +2,10 @@ #adapted from https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/sql/postgresql/ext/pgaudit.nix let source = { + "17" = { + version = "17.0"; + hash = "sha256-3ksq09wiudQPuBQI3dhEQi8IkXKLVIsPFgBnwLiicro="; + }; "16" = { version = "16.0"; hash = "sha256-8+tGOl1U5y9Zgu+9O5UDDE4bec4B0JC/BQ6GLhHzQzc="; diff --git a/nix/ext/pgroonga.nix b/nix/ext/pgroonga.nix index 884b54582..c7fe90bb1 100644 --- a/nix/ext/pgroonga.nix +++ b/nix/ext/pgroonga.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper }: +{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper, xxHash }: let supabase-groonga = callPackage ../supabase-groonga.nix { }; in @@ -10,7 +10,11 @@ stdenv.mkDerivation rec { sha256 = "sha256-iF/zh4zDDpAw5fxW1WG8i2bfPt4VYsnYArwOoE/lwgM="; }; nativeBuildInputs = [ pkg-config makeWrapper ]; - buildInputs = [ postgresql msgpack-c supabase-groonga mecab ]; + + buildInputs = [ postgresql msgpack-c supabase-groonga mecab ] ++ lib.optionals stdenv.isDarwin [ + xxHash + ]; + propagatedBuildInputs = [ supabase-groonga ]; configureFlags = [ "--with-mecab=${mecab}" @@ -19,16 +23,30 @@ stdenv.mkDerivation rec { "--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins" ]; - makeFlags = [ + makeFlags = [ "HAVE_MSGPACK=1" "MSGPACK_PACKAGE_NAME=msgpack-c" "HAVE_MECAB=1" ]; + NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin (builtins.concatStringsSep " " [ + "-Wno-error=incompatible-function-pointer-types" + "-Wno-error=format" + "-Wno-format" + "-I${supabase-groonga}/include/groonga" + "-I${xxHash}/include" + "-DPGRN_VERSION=\"${version}\"" + ]); + preConfigure = '' export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga" export GROONGA_CFLAGS="-I${supabase-groonga}/include" export MECAB_CONFIG="${mecab}/bin/mecab-config" + ${lib.optionalString stdenv.isDarwin '' + export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format" + ''} ''; installPhase = '' diff --git a/nix/ext/rum.nix b/nix/ext/rum.nix index 16bf106c5..1270f1ff1 100644 --- a/nix/ext/rum.nix +++ b/nix/ext/rum.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "rum"; - version = "1.3.13"; + version = "1.3.14"; src = fetchFromGitHub { owner = "postgrespro"; repo = "rum"; rev = version; - hash = "sha256-yy2xeDnk3fENN+En0st4mv60nZlqPafIzwf68jwJ5fE="; + hash = "sha256-VsfpxQqRBu9bIAP+TfMRXd+B3hSjuhU2NsutocNiCt8="; }; buildInputs = [ postgresql ]; diff --git a/nix/ext/wal2json.nix b/nix/ext/wal2json.nix index 751eb64c0..ed578c7fd 100644 --- a/nix/ext/wal2json.nix +++ b/nix/ext/wal2json.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "wal2json"; - version = "2_5"; + version = "2_6"; src = fetchFromGitHub { owner = "eulerto"; repo = "wal2json"; rev = "wal2json_${builtins.replaceStrings ["."] ["_"] version}"; - hash = "sha256-Gpc9uDKrs/dmVSFgdgHM453+TaEnhRh9t0gDbSn8FUI="; + hash = "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM="; }; buildInputs = [ postgresql ]; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 54f2c275b..7125f41be 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -50,6 +50,7 @@ buildPgrxExtension_0_12_6 rec { CARGO_BUILD_JOBS = "2"; CARGO="${cargo}/bin/cargo"; + #CARGO_NET_GIT_FETCH_WITH_CLI = "true"; cargoLock = { lockFile = "${src}/Cargo.lock"; allowBuiltinFetchGit = false; @@ -57,6 +58,41 @@ buildPgrxExtension_0_12_6 rec { "clickhouse-rs-1.1.0-alpha.1" = "sha256-G+v4lNP5eK2U45D1fL90Dq24pUSlpIysNCxuZ17eac0="; }; }; + + preConfigure = '' + cd wrappers + + # update the clickhouse-rs dependency + # append the branch name to the git URL to help cargo locate the commit + # while maintaining the rev for reproducibility + awk -i inplace ' + /\[dependencies.clickhouse-rs\]/ { + print + getline + if ($0 ~ /git =/) { + print "git = \"https://github.com/suharev7/clickhouse-rs/async-await\"" + } else { + print + } + while ($0 !~ /^\[/ && NF > 0) { + getline + if ($0 ~ /rev =/) print + if ($0 ~ /^\[/) print + } + next + } + { print } + ' Cargo.toml + + # Verify the file is still valid TOML, break build with this error + # if it is not + if ! cargo verify-project 2>/dev/null; then + echo "Failed to maintain valid TOML syntax" + exit 1 + fi + + cd .. + ''; buildAndTestSubdir = "wrappers"; buildFeatures = [ diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index d55af10a2..309129f1f 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { - postgresql_16 = prev.postgresql_16.overrideAttrs (old: { - pname = "postgresql_16"; - version = "16_23"; + pg_orioledb = prev.postgresql_16.overrideAttrs (old: { + pname = "postgresql_orioledb"; + version = "16_31"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz"; - sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; + sha256 = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; }; buildInputs = old.buildInputs ++ [ prev.bison @@ -17,5 +17,5 @@ final: prev: { prev.perl ]; }); - postgresql_orioledb_16 = final.postgresql_16; + postgresql_orioledb = final.pg_orioledb; } diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 5f92262b2..a065d5aa7 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -1,11 +1,11 @@ self: let - #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; postgresql_16 = ./16.nix; + postgresql_orioledb-16 = ./orioledb-16.nix; + postgresql_orioledb-17 = ./orioledb-17.nix; }; - mkAttributes = jitSupport: self.lib.mapAttrs' (version: path: let @@ -15,7 +15,6 @@ let inherit jitSupport self; }) ) versions; - in # variations without and with JIT (mkAttributes false) // (mkAttributes true) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 323b65167..19b73c162 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -7,6 +7,8 @@ let , glibc, zlib, readline, openssl, icu, lz4, zstd, systemd, libossp_uuid , pkg-config, libxml2, tzdata, libkrb5, substituteAll, darwin , linux-pam + #orioledb specific + , perl, bison, flex, docbook_xsl, docbook_xml_dtd_45, docbook_xsl_ns, libxslt # This is important to obtain a version of `libpq` that does not depend on systemd. , systemdSupport ? lib.meta.availableOn stdenv.hostPlatform systemd && !stdenv.hostPlatform.isStatic @@ -49,14 +51,20 @@ let inherit version; pname = pname + lib.optionalString jitSupport "-jit"; - src = fetchurl { - url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; - inherit hash; - }; + src = if (builtins.match "[0-9][0-9]_.*" version != null) then + fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; + inherit hash; + } + else + fetchurl { + url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; + inherit hash; + }; hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; - outputs = [ "out" "lib" "doc" "man" ]; + outputs = [ "out" "lib" ]; setOutputFlags = false; # $out retains configureFlags :-/ buildInputs = [ @@ -74,7 +82,10 @@ let ++ lib.optionals pythonSupport [ python3 ] ++ lib.optionals gssSupport [ libkrb5 ] ++ lib.optionals stdenv'.isLinux [ linux-pam ] - ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ]; + ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] + ++ lib.optionals (builtins.match "[0-9][0-9]_.*" version != null) [ + perl bison flex docbook_xsl docbook_xml_dtd_45 docbook_xsl_ns libxslt + ]; nativeBuildInputs = [ makeWrapper @@ -86,7 +97,7 @@ let separateDebugInfo = true; - buildFlags = [ "world" ]; + buildFlags = [ "world-bin" ]; # Makes cross-compiling work when xml2-config can't be executed on the host. # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 @@ -127,7 +138,7 @@ let (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch) ]; - installTargets = [ "install-world" ]; + installTargets = [ "install-world-bin" ]; postPatch = '' # Hardcode the path to pgxs so pg_config returns the path in $out @@ -277,7 +288,7 @@ let paths = f pkgs ++ [ postgresql postgresql.lib - postgresql.man # in case user installs this into environment + #TODO RM postgresql.man # in case user installs this into environment ]; nativeBuildInputs = [ makeWrapper ]; diff --git a/nix/postgresql/orioledb-16.nix b/nix/postgresql/orioledb-16.nix new file mode 100644 index 000000000..e9a66850e --- /dev/null +++ b/nix/postgresql/orioledb-16.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "16_31"; + hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; +} diff --git a/nix/postgresql/orioledb-17.nix b/nix/postgresql/orioledb-17.nix new file mode 100644 index 000000000..bbe5024a5 --- /dev/null +++ b/nix/postgresql/orioledb-17.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "17_4"; + hash = "sha256-qMJpTcw/ExipQ1XIPvYTQK+qgcDC7zecdLJDIuQug+0="; +} diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in index 309588828..3c4e9181e 100644 --- a/nix/tools/run-client.sh.in +++ b/nix/tools/run-client.sh.in @@ -92,9 +92,9 @@ elif [ "$PSQL_VERSION" == "16" ]; then echo "Starting client for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" -elif [ "$PSQL_VERSION" == "orioledb-16" ]; then - echo "Starting client for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ +elif [ "$PSQL_VERSION" == "orioledb-17" ]; then + echo "Starting client for PSQL ORIOLEDB 17" + PSQLORIOLEDB16=@PSQLORIOLEDB17_BINDIR@ BINDIR="$PSQLORIOLEDB16" else echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 2b079bbf1..1ebae228f 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -1,7 +1,7 @@ #!/usr/bin/env bash # shellcheck shell=bash [ ! -z "$DEBUG" ] && set -x -# first argument should be '15' or '16' for the version + if [ "$1" == "15" ]; then echo "Starting server for PSQL 15" PSQL15=@PSQL15_BINDIR@ @@ -10,10 +10,10 @@ elif [ "$1" == "16" ]; then echo "Starting server for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" -elif [ "$1" == "orioledb-16" ]; then - echo "Starting server for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ - BINDIR="$PSQLORIOLEDB16" +elif [ "$1" == "orioledb-17" ]; then + echo "Starting server for PSQL ORIOLEDB 17" + PSQLORIOLEDB17=@PSQLORIOLEDB17_BINDIR@ + BINDIR="$PSQLORIOLEDB17" else echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" exit 1 @@ -32,6 +32,7 @@ EXTENSION_CUSTOM_SCRIPTS=@EXTENSION_CUSTOM_SCRIPTS_DIR@ GROONGA=@GROONGA_DIR@ DATDIR=$(mktemp -d) LOCALE_ARCHIVE=@LOCALES@ +CURRENT_SYSTEM=@CURRENT_SYSTEM@ export LOCALE_ARCHIVE export LANG=en_US.UTF-8 export LANGUAGE=en_US.UTF-8 @@ -60,5 +61,29 @@ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "\$a\\ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" + +orioledb_config_items() { + if [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" != "aarch64-darwin" ]]; then + # Remove items from postgresql.conf + sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" + sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATADIR/postgresql.conf" + echo "default_table_access_method = 'orioledb'" >> "$DATADIR/postgresql.conf" + + fi + if [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" = "aarch64-darwin" ]]; then + # Remove items from postgresql.conf + sed -i '' 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present + sed -i '' 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" + sed -i '' 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + sed -i '' 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATADIR/postgresql.conf" + echo "default_table_access_method = 'orioledb'" >> "$DATADIR/postgresql.conf" + + fi +} +orioledb_config_items "$1" export GRN_PLUGINS_DIR=$GROONGA/lib/groonga/plugins postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp