From cdc64524f25374081f2541428e9c89a4429ab6cd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:28:53 -0400 Subject: [PATCH 001/109] feat: build and flake check of pg 16.3 with exts/wrappers --- flake.nix | 35 ++++++++-- nix/ext/hypopg.nix | 2 +- nix/ext/pg_backtrace.nix | 2 +- nix/ext/pg_cron.nix | 2 +- nix/ext/pg_hashids.nix | 2 +- nix/ext/pg_net.nix | 2 +- nix/ext/pg_plan_filter.nix | 2 +- nix/ext/pg_stat_monitor.nix | 2 +- nix/ext/pg_tle.nix | 2 +- nix/ext/pgrouting.nix | 33 +++++++++- nix/ext/pgsodium.nix | 2 +- nix/ext/pgsql-http.nix | 2 +- nix/ext/pgvector.nix | 2 +- nix/ext/plv8.nix | 76 ++-------------------- nix/ext/supautils.nix | 2 +- nix/postgresql/16.nix | 4 ++ nix/postgresql/default.nix | 1 + nix/tests/sql/extensions_sql_interface.sql | 12 ++++ 18 files changed, 96 insertions(+), 89 deletions(-) create mode 100644 nix/postgresql/16.nix diff --git a/flake.nix b/flake.nix index 82409b185..3bb6d1a45 100644 --- a/flake.nix +++ b/flake.nix @@ -290,8 +290,7 @@ # Define the available PostgreSQL versions postgresVersions = { psql_15 = makePostgres "15"; - # Uncomment the line below to enable PostgreSQL 16 - # psql_16 = makePostgres "16"; + psql_16 = makePostgres "16"; # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; }; @@ -307,15 +306,17 @@ postgresql = postgresqlPackage; }; postgresql_15 = getPostgresqlPackage "15"; + postgresql_16 = getPostgresqlPackage "16"; in postgresVersions //{ supabase-groonga = supabase-groonga; # PostgreSQL versions. - psql_15 = makePostgres "15"; + psql_15 = postgresVersions.psql_15; + psql_16 = postgresVersions.psql_16; #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15; + inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; @@ -339,6 +340,28 @@ platforms = platforms.all; }; }; + postgresql_16_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-16-src"; + version = postgresql_16.version; + + src = postgresql_16.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; @@ -397,6 +420,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -425,6 +449,7 @@ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ + --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ @@ -569,7 +594,7 @@ # flake check'. This is run in the CI system, as well. checks = { psql_15 = makeCheckHarness basePackages.psql_15.bin; - #psql_16 = makeCheckHarness basePackages.psql_16.bin; + psql_16 = makeCheckHarness basePackages.psql_16.bin; #psql_orioledb_16 = makeCheckHarness basePackages.psql_orioledb_16.bin; }; diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index 4fc00a82f..300a44900 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_backtrace.nix b/nix/ext/pg_backtrace.nix index b0169124c..47ede88e2 100644 --- a/nix/ext/pg_backtrace.nix +++ b/nix/ext/pg_backtrace.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 5c546c797..d51254aa2 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 41c3ba664..50bff95da 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..c0946f425 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 2d7d224ec..0ed527251 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 8784067d2..c3e91fe2b 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index 6b1c7b1ca..0221d8768 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -21,7 +21,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index 36edf3404..5587566cd 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -14,9 +14,38 @@ stdenv.mkDerivation rec { hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; }; + #disable compile time warnings for incompatible pointer types only on macos and pg16 + NIX_CFLAGS_COMPILE = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") + "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; + + cmakeFlags = [ + "-DPOSTGRESQL_VERSION=${postgresql.version}" + ] ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ + "-DCMAKE_MACOSX_RPATH=ON" + "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" + "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" + ]; + + preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + export DLSUFFIX=.dylib + export CMAKE_SHARED_LIBRARY_SUFFIX=.dylib + export CMAKE_SHARED_MODULE_SUFFIX=.dylib + export MACOSX_RPATH=ON + ''; + + postBuild = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' + shopt -s nullglob + for file in lib/libpgrouting-*.so; do + if [ -f "$file" ]; then + mv "$file" "''${file%.so}.dylib" + fi + done + shopt -u nullglob + ''; + installPhase = '' - install -D lib/*.so -t $out/lib - install -D sql/pgrouting--${version}.sql -t $out/share/postgresql/extension + install -D lib/*${postgresql.dlSuffix} -t $out/lib + install -D sql/pgrouting--*.sql -t $out/share/postgresql/extension install -D sql/common/pgrouting.control -t $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index e3b0da228..4e184faa9 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 2edaa9d53..de8fa68c6 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 476f7606c..aefeeea3a 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..785f5e601 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -105,81 +105,17 @@ stdenv.mkDerivation (finalAttrs: { rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} ''; - passthru = { - tests = - let - postgresqlWithSelf = postgresql.withPackages (_: [ - finalAttrs.finalPackage - ]); - in { - smoke = runCommand "plv8-smoke-test" {} '' - export PATH=${lib.makeBinPath [ - postgresqlWithSelf - coreutils - gnugrep - ]} - db="$PWD/testdb" - initdb "$db" - postgres -k "$db" -D "$db" & - pid="$!" - - for i in $(seq 1 100); do - if psql -h "$db" -d postgres -c "" 2>/dev/null; then - break - elif ! kill -0 "$pid"; then - exit 1 - else - sleep 0.1 - fi - done - - psql -h "$db" -d postgres -c 'CREATE EXTENSION plv8; DO $$ plv8.elog(NOTICE, plv8.version); $$ LANGUAGE plv8;' 2> "$out" - grep -q "${finalAttrs.version}" "$out" - kill -0 "$pid" - ''; - - regression = stdenv.mkDerivation { - name = "plv8-regression"; - inherit (finalAttrs) src patches nativeBuildInputs buildInputs dontConfigure; - - buildPhase = '' - runHook preBuild - - # The regression tests need to be run in the order specified in the Makefile. - echo -e "include Makefile\nprint_regress_files:\n\t@echo \$(REGRESS)" > Makefile.regress - REGRESS_TESTS=$(make -f Makefile.regress print_regress_files) - - ${postgresql}/lib/pgxs/src/test/regress/pg_regress \ - --bindir='${postgresqlWithSelf}/bin' \ - --temp-instance=regress-instance \ - --dbname=contrib_regression \ - $REGRESS_TESTS - - runHook postBuild - ''; - - installPhase = '' - runHook preInstall - - touch "$out" - - runHook postInstall - ''; - }; - }; - }; - meta = with lib; { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; @@ -187,4 +123,4 @@ stdenv.mkDerivation (finalAttrs: { platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" ]; license = licenses.postgresql; }; -}) \ No newline at end of file +}) diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..ee1d691b2 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -16,7 +16,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/lib - install -D supautils.so -t $out/lib + install -D *${postgresql.dlSuffix} -t $out/lib ''; meta = with lib; { diff --git a/nix/postgresql/16.nix b/nix/postgresql/16.nix new file mode 100644 index 000000000..c964fc580 --- /dev/null +++ b/nix/postgresql/16.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "16.3"; + hash = "sha256-Mxlj1dPcTK9CFqBJ+kC2bWvLjHMGFYWUEblRh2TmBYU="; +} diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 6ee0452dd..5f92262b2 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -3,6 +3,7 @@ let #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; + postgresql_16 = ./16.nix; }; mkAttributes = jitSupport: diff --git a/nix/tests/sql/extensions_sql_interface.sql b/nix/tests/sql/extensions_sql_interface.sql index 947a4a612..ad0f63ed8 100644 --- a/nix/tests/sql/extensions_sql_interface.sql +++ b/nix/tests/sql/extensions_sql_interface.sql @@ -63,6 +63,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, From 91ec362374e3ec73ada1f7215daa2a0c2c0de236 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:44:54 -0500 Subject: [PATCH 002/109] pg_partman test 15/16 compat --- nix/tests/expected/pg_partman.out | 5 ++++- nix/tests/sql/pg_partman.sql | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nix/tests/expected/pg_partman.out b/nix/tests/expected/pg_partman.out index 527d58b12..25aa80d2e 100644 --- a/nix/tests/expected/pg_partman.out +++ b/nix/tests/expected/pg_partman.out @@ -28,7 +28,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; table_name | table_type -----------------------------+------------ time_taptest_table | BASE TABLE diff --git a/nix/tests/sql/pg_partman.sql b/nix/tests/sql/pg_partman.sql index 621e1a7c9..d1fb9f4ba 100644 --- a/nix/tests/sql/pg_partman.sql +++ b/nix/tests/sql/pg_partman.sql @@ -35,7 +35,10 @@ select from information_schema.tables where - table_schema = 'partman_test'; + table_schema = 'partman_test' +order by + table_name, + table_type; select public.create_parent( From 0321dfa85e67b274f06a3dd705aa6407d8f8cace Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 7 Oct 2024 15:46:33 -0500 Subject: [PATCH 003/109] merge sql interface test --- nix/tests/expected/extensions_sql_interface.out | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 5714fbcec..07c0d6650 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -146,6 +146,18 @@ from on e.oid = d.refobjid where d.deptype = 'e' + -- Filter out changes between pg15 and pg16 from extensions that ship with postgres + -- new in pg16 + and not (e.extname = 'fuzzystrmatch' and p.proname = 'daitch_mokotoff') + and not (e.extname = 'pageinspect' and p.proname = 'bt_multi_page_stats') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_summary') + and not (e.extname = 'pg_buffercache' and p.proname = 'pg_buffercache_usage_counts') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_block_info') + -- removed in pg16 + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_records_info_till_end_of_wal') + and not (e.extname = 'pg_walinspect' and p.proname = 'pg_get_wal_stats_till_end_of_wal') + -- changed in pg16 - output signature added a column + and not (e.extname = 'pageinspect' and p.proname = 'brin_page_items') order by e.extname, n.nspname, @@ -1076,7 +1088,6 @@ order by moddatetime | public | moddatetime | | trigger old_snapshot | public | pg_old_snapshot_time_mapping | OUT array_offset integer, OUT end_timestamp timestamp with time zone, OUT newest_xmin xid | SETOF record pageinspect | public | brin_metapage_info | page bytea, OUT magic text, OUT version integer, OUT pagesperrange integer, OUT lastrevmappage bigint | record - pageinspect | public | brin_page_items | page bytea, index_oid regclass, OUT itemoffset integer, OUT blknum bigint, OUT attnum integer, OUT allnulls boolean, OUT hasnulls boolean, OUT placeholder boolean, OUT value text | SETOF record pageinspect | public | brin_page_type | page bytea | text pageinspect | public | brin_revmap_data | page bytea, OUT pages tid | SETOF tid pageinspect | public | bt_metap | relname text, OUT magic integer, OUT version integer, OUT root bigint, OUT level bigint, OUT fastroot bigint, OUT fastlevel bigint, OUT last_cleanup_num_delpages bigint, OUT last_cleanup_num_tuples double precision, OUT allequalimage boolean | record @@ -1300,9 +1311,7 @@ order by pg_visibility | public | pg_visibility_map_summary | regclass, OUT all_visible bigint, OUT all_frozen bigint | record pg_walinspect | public | pg_get_wal_record_info | in_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | record pg_walinspect | public | pg_get_wal_records_info | start_lsn pg_lsn, end_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record - pg_walinspect | public | pg_get_wal_records_info_till_end_of_wal | start_lsn pg_lsn, OUT start_lsn pg_lsn, OUT end_lsn pg_lsn, OUT prev_lsn pg_lsn, OUT xid xid, OUT resource_manager text, OUT record_type text, OUT record_length integer, OUT main_data_length integer, OUT fpi_length integer, OUT description text, OUT block_ref text | SETOF record pg_walinspect | public | pg_get_wal_stats | start_lsn pg_lsn, end_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record - pg_walinspect | public | pg_get_wal_stats_till_end_of_wal | start_lsn pg_lsn, per_record boolean, OUT "resource_manager/record_type" text, OUT count bigint, OUT count_percentage double precision, OUT record_size bigint, OUT record_size_percentage double precision, OUT fpi_size bigint, OUT fpi_size_percentage double precision, OUT combined_size bigint, OUT combined_size_percentage double precision | SETOF record pgaudit | public | pgaudit_ddl_command_end | | event_trigger pgaudit | public | pgaudit_sql_drop | | event_trigger pgcrypto | public | armor | bytea | text @@ -5225,7 +5234,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5072 rows) +(5069 rows) /* From aa5dd9a271eb4c79bf97e77e3e92d628900c281e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 16:57:53 -0400 Subject: [PATCH 004/109] tests: build test and cache both versions --- docker/nix/build_nix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index a8a8639e2..3aba72c2c 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -6,5 +6,7 @@ if [ -d "/workspace" ]; then cd /workspace fi nix build .#psql_15/bin -o psql_15 +nix build .#psql_16/bin -o psql_16 nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From 65fdc5d97a00ac46a025b990f149333f51e5a052 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 7 Oct 2024 18:31:02 -0400 Subject: [PATCH 005/109] chore: run checks individually --- docker/nix/build_nix.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3aba72c2c..3cf8505c0 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,8 +5,9 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 From f10881b512b5e0497f35e7d3ed59169098f1daca Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 10 Oct 2024 13:01:02 -0400 Subject: [PATCH 006/109] pg 15 and 16 packer/ansible/ghactions (#1268) * fix: reformat ec2 cleanup commands (#1267) Co-authored-by: Sam Rose * feat: build and flake check of pg 16.3 with exts/wrappers * pg_partman test 15/16 compat * merge sql interface test * tests: build test and cache both versions * chore: run checks individually * feat: realease 15 and 16 to staging * chore: update versions * chore: make yq available * chore: run yq from nix * chore: more setup for staging AMI * fix: yq usage * chore: shell vars * fix: When --init none is used, only users who can elevate to sudo privileges can run Nix * fix: no -i * fix: quote correction * fix: newline extra quotes * fix: no need for pg major version on packer * fix: postgresql_major * fix: ql * fix: no ansible args in stage to invocation * fix: unique val * fix: adjustments to build scripts * chore: env var handling * fix: bump to build * chore: set up more required vars * chore: bump var * feat: pg 16 debug symbols * feat: matrix pg versions build on testinfra * feat: matrix on Test Database * chore: running nix in the right context * feat: just use existing Dockerfile + pg version * chore: refer to var * fix: read name without including quotes * chore: try format function * fix: strip quotes from version number * chore: env var * fix: pg client * fix * fix: try to use psql from our own corresponding pkg * fix: try psql from ppa * fix: dbmate per pg version * build dbmate and then install client * fix: account for architecture * chore: limit changes detection migrations/schema.sql * missing docker compose call * ore: drop tests while investigating * test: try on pg15 only * chore: schema needs update * chore: now run on all versions in matrix * test: trying a version of schema per major pg version as there are type diffs --------- Co-authored-by: Sam Rose Co-authored-by: Oliver Rice --- .github/workflows/ami-release-nix.yml | 65 +- .github/workflows/test.yml | 65 +- .github/workflows/testinfra-nix.yml | 45 +- Dockerfile-15 | 223 +++++ Dockerfile-16 | 223 +++++ ansible/tasks/stage2-setup-postgres.yml | 14 +- ansible/vars.yml | 14 +- common-nix.vars.pkr.hcl | 1 - flake.nix | 1 + migrations/Dockerfile.dbmate | 23 + migrations/docker-compose.yaml | 4 +- migrations/schema-15.sql | 1064 +++++++++++++++++++++++ migrations/schema-16.sql | 1064 +++++++++++++++++++++++ migrations/schema.sql | 2 +- scripts/nix-provision.sh | 7 + stage2-nix-psql.pkr.hcl | 8 +- 16 files changed, 2780 insertions(+), 43 deletions(-) create mode 100644 Dockerfile-15 create mode 100644 Dockerfile-16 delete mode 100644 common-nix.vars.pkr.hcl create mode 100644 migrations/Dockerfile.dbmate create mode 100644 migrations/schema-15.sql create mode 100644 migrations/schema-16.sql diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 194c8de21..643f26fc7 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -8,12 +8,31 @@ on: paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' + - 'ansible/vars.yml' workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 @@ -31,42 +50,55 @@ jobs: - name: Checkout Repo uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Run checks if triggered manually if: ${{ github.event_name == 'workflow_dispatch' }} - # Update `ci.yaml` too if changing constraints. run: | - SUFFIX=$(sed -E 's/postgres-version = "[0-9\.]+(.*)"/\1/g' common-nix.vars.pkr.hcl) + SUFFIX=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres${{ matrix.postgres_version }}"]' ansible/vars.yml | sed -E 's/[0-9\.]+(.*)$/\1/') if [[ -z $SUFFIX ]] ; then echo "Version must include non-numeric characters if built manually." exit 1 fi - # extensions are build in nix prior to this step - # so we can just use the binaries from the nix store - # for postgres, extensions and wrappers + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl - name: Grab release version id: process_release_version run: | - VERSION=$(sed -e 's/postgres-version = "\(.*\)"/\1/g' common-nix.vars.pkr.hcl) - echo "version=$VERSION" >> "$GITHUB_OUTPUT" + VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + echo "version=$VERSION" >> $GITHUB_OUTPUT - name: Create nix flake revision tarball run: | GIT_SHA=${{github.sha}} - MAJOR_VERSION=$(echo "${{ steps.process_release_version.outputs.version }}" | cut -d. -f1) + MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" @@ -84,17 +116,13 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 staging run: | aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz - #Our self hosted github runner already has permissions to publish images - #but they're limited to only that; - #so if we want s3 access we'll need to config credentials with the below steps - # (which overwrites existing perms) after the ami build - - name: configure aws credentials - prod uses: aws-actions/configure-aws-credentials@v4 with: @@ -107,6 +135,7 @@ jobs: ansible-playbook -i localhost \ -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ manifest-playbook.yml - name: Upload nix flake revision to s3 prod @@ -130,12 +159,12 @@ jobs: SLACK_MESSAGE: 'Building Postgres AMI failed' SLACK_FOOTER: '' - - name: Cleanup resources on build cancellation + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --instance-ids {} + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2be061941..1457eb100 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,27 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + build: + needs: prepare strategy: matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: [self-hosted, X64] arch: amd64 @@ -23,14 +41,36 @@ jobs: POSTGRES_PASSWORD: password steps: - uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Strip quotes from pg major and set env var + run: | + stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') + echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + - id: settings # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - id: args - uses: mikefarah/yq@master - with: - cmd: yq 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' 'ansible/vars.yml' + - name: Generate args + id: args + run: | + ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) + echo "result<> $GITHUB_OUTPUT + echo "$ARGS" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT - run: docker context create builders - uses: docker/setup-buildx-action@v3 @@ -40,7 +80,7 @@ jobs: with: load: true context: . - file: "Dockerfile-156" + file: Dockerfile-${{ env.PGMAJOR }} target: production build-args: | ${{ steps.args.outputs.result }} @@ -57,10 +97,13 @@ jobs: -p ${{ env.POSTGRES_PORT }}:5432 \ --name supabase_postgres \ -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} + - name: Install psql run: | + sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt update - sudo apt install -y --no-install-recommends postgresql-client + sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP @@ -107,11 +150,15 @@ jobs: PGUSER: supabase_admin PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + - name: Update Dockerfile.dbmate version + run: | + sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate + - name: verify schema.sql is committed run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit - if ! git diff --ignore-space-at-eol --exit-code --quiet migrations/schema.sql; then - echo "Detected uncommitted changes after build. See status below:" - git diff + if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then + echo "Detected changes in schema.sql:" + git diff migrations/schema-${{ env.PGMAJOR }}.sql exit 1 fi diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 3835a9a00..4a51c159c 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -5,17 +5,35 @@ on: workflow_dispatch: jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + test-ami-nix: + needs: prepare strategy: fail-fast: false matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} include: - runner: arm-runner arch: arm64 ubuntu_release: focal ubuntu_version: 20.04 mcpu: neoverse-n1 - runs-on: ${{ matrix.runner }} + runs-on: ${{ matrix.runner }} timeout-minutes: 150 permissions: contents: write @@ -40,18 +58,29 @@ jobs: - name: Generate random string id: random run: echo "random_string=$(openssl rand -hex 8)" >> $GITHUB_OUTPUT + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl - name: Build AMI stage 1 run: | packer init amazon-arm64-nix.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" amazon-arm64-nix.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 run: | packer init stage2-nix-psql.pkr.hcl GIT_SHA=${{github.sha}} - packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "git_sha=${GITHUB_SHA}" stage2-nix-psql.pkr.hcl - name: Run tests timeout-minutes: 10 @@ -65,12 +94,12 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} - - - name: Cleanup resources on build cancellation + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids + + - name: Cleanup resources after build if: ${{ always() }} run: | - aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -n 1 -I {} aws ec2 terminate-instances --region ap-southeast-1 --instance-ids {} || true + aws ec2 --region ap-southeast-1 describe-instances --filters "Name=tag:testinfra-run-id,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region ap-southeast-1 --instance-ids || true - name: Cleanup AMIs if: always() @@ -91,4 +120,4 @@ jobs: # Deregister AMIs deregister_ami_by_name "$STAGE1_AMI_NAME" - deregister_ami_by_name "$STAGE2_AMI_NAME" \ No newline at end of file + deregister_ami_by_name "$STAGE2_AMI_NAME" diff --git a/Dockerfile-15 b/Dockerfile-15 new file mode 100644 index 000000000..ce83237f2 --- /dev/null +++ b/Dockerfile-15 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_15/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/Dockerfile-16 b/Dockerfile-16 new file mode 100644 index 000000000..db5772cee --- /dev/null +++ b/Dockerfile-16 @@ -0,0 +1,223 @@ +# syntax=docker/dockerfile:1.6 +ARG postgresql_major=15 +ARG postgresql_release=${postgresql_major}.1 + +# Bump default build arg to build a package from source +# Bump vars.yml to specify runtime package version +ARG sfcgal_release=1.3.10 +ARG postgis_release=3.3.2 +ARG pgrouting_release=3.4.1 +ARG pgtap_release=1.2.0 +ARG pg_cron_release=1.6.2 +ARG pgaudit_release=1.7.0 +ARG pgjwt_release=9742dab1b2f297ad3811120db7b21451bca2d3c9 +ARG pgsql_http_release=1.5.0 +ARG plpgsql_check_release=2.2.5 +ARG pg_safeupdate_release=1.4 +ARG timescaledb_release=2.9.1 +ARG wal2json_release=2_5 +ARG pljava_release=1.6.4 +ARG plv8_release=3.1.5 +ARG pg_plan_filter_release=5081a7b5cb890876e67d8e7486b6a64c38c9a492 +ARG pg_net_release=0.7.1 +ARG rum_release=1.3.13 +ARG pg_hashids_release=cd0e1b31d52b394a0df64079406a14a4f7387cd6 +ARG libsodium_release=1.0.18 +ARG pgsodium_release=3.1.6 +ARG pg_graphql_release=1.5.1 +ARG pg_stat_monitor_release=1.1.1 +ARG pg_jsonschema_release=0.1.4 +ARG pg_repack_release=1.4.8 +ARG vault_release=0.2.8 +ARG groonga_release=12.0.8 +ARG pgroonga_release=2.4.0 +ARG wrappers_release=0.3.0 +ARG hypopg_release=1.3.1 +ARG pgvector_release=0.4.0 +ARG pg_tle_release=1.3.2 +ARG index_advisor_release=0.2.0 +ARG supautils_release=2.2.0 +ARG wal_g_release=2.0.1 + +FROM ubuntu:focal as base + +RUN apt update -y && apt install -y \ + curl \ + gnupg \ + lsb-release \ + software-properties-common \ + wget \ + sudo \ + && apt clean + + +RUN adduser --system --home /var/lib/postgresql --no-create-home --shell /bin/bash --group --gecos "PostgreSQL administrator" postgres +RUN adduser --system --no-create-home --shell /bin/bash --group wal-g +RUN curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install linux \ +--init none \ +--no-confirm \ +--extra-conf "substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com" \ +--extra-conf "trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + +ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" + +COPY . /nixpg + +WORKDIR /nixpg + +RUN nix profile install .#psql_16/bin + + + +WORKDIR / + + +RUN mkdir -p /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + # /usr/lib/postgresql/share/postgresql/contrib \ + #/usr/lib/postgresql/share/postgresql/timezonesets \ + #/usr/lib/postgresql/share/postgresql/tsearch_data \ + # /usr/lib/postgresql/share/postgresql/extension \ + && chown -R postgres:postgres /usr/lib/postgresql \ + && chown -R postgres:postgres /usr/share/postgresql + +# Create symbolic links +RUN ln -s /nix/var/nix/profiles/default/bin/* /usr/lib/postgresql/bin/ \ + && ln -s /nix/var/nix/profiles/default/bin/* /usr/bin/ \ + && chown -R postgres:postgres /usr/bin + +# Create symbolic links for PostgreSQL shares +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ +RUN ln -s /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ +RUN chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/ +RUN chown -R postgres:postgres /usr/share/postgresql/ +# Create symbolic links for contrib directory +RUN mkdir -p /usr/lib/postgresql/share/postgresql/contrib \ + && find /nix/var/nix/profiles/default/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres /usr/lib/postgresql/share/postgresql/contrib/ + +RUN chown -R postgres:postgres /usr/lib/postgresql + +RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets + + +RUN apt-get update && \ + apt-get install -y --no-install-recommends tzdata + +RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \ + dpkg-reconfigure --frontend noninteractive tzdata + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + checkinstall \ + cmake + +ENV PGDATA=/var/lib/postgresql/data + +#################### +# setup-wal-g.yml +#################### +FROM base as walg +ARG wal_g_release +# ADD "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-${TARGETARCH}.tar.gz" /tmp/wal-g.tar.gz +RUN arch=$([ "$TARGETARCH" = "arm64" ] && echo "aarch64" || echo "$TARGETARCH") && \ + apt-get update && apt-get install -y --no-install-recommends curl && \ + curl -kL "https://github.com/wal-g/wal-g/releases/download/v${wal_g_release}/wal-g-pg-ubuntu-20.04-aarch64.tar.gz" -o /tmp/wal-g.tar.gz && \ + tar -xvf /tmp/wal-g.tar.gz -C /tmp && \ + rm -rf /tmp/wal-g.tar.gz && \ + mv /tmp/wal-g-pg-ubuntu*20.04-aarch64 /tmp/wal-g + +# #################### +# # Download gosu for easy step-down from root +# #################### +FROM base as gosu +ARG TARGETARCH +# Install dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gnupg \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* +# Download binary +ARG GOSU_VERSION=1.16 +ARG GOSU_GPG_KEY=B42F6819007F00F88E364FD4036A9C25BF357DD4 +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH \ + /usr/local/bin/gosu +ADD https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$TARGETARCH.asc \ + /usr/local/bin/gosu.asc +# Verify checksum +RUN gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys $GOSU_GPG_KEY && \ + gpg --batch --verify /usr/local/bin/gosu.asc /usr/local/bin/gosu && \ + gpgconf --kill all && \ + chmod +x /usr/local/bin/gosu + +# #################### +# # Build final image +# #################### +FROM gosu as production +RUN id postgres || (echo "postgres user does not exist" && exit 1) +# # Setup extensions +COPY --from=walg /tmp/wal-g /usr/local/bin/ + +# # Initialise configs +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf.j2 /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf.j2 /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh +COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh + +RUN sed -i \ + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ + echo "cron.database_name = 'postgres'" >> /etc/postgresql/postgresql.conf && \ + #echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \ + usermod -aG postgres wal-g && \ + mkdir -p /etc/postgresql-custom && \ + chown postgres:postgres /etc/postgresql-custom + +# # Include schema migrations +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +# # Add upstream entrypoint script +COPY --from=gosu /usr/local/bin/gosu /usr/local/bin/gosu +ADD --chmod=0755 \ + https://github.com/docker-library/postgres/raw/master/15/bullseye/docker-entrypoint.sh \ + /usr/local/bin/ + +RUN mkdir -p /var/run/postgresql && chown postgres:postgres /var/run/postgresql + +ENTRYPOINT ["docker-entrypoint.sh"] + +HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost +STOPSIGNAL SIGINT +EXPOSE 5432 + +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +RUN apt-get update && apt-get install -y --no-install-recommends \ + locales \ + && rm -rf /var/lib/apt/lists/* && \ + localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \ + && localedef -i C -c -f UTF-8 -A /usr/share/locale/locale.alias C.UTF-8 +RUN echo "C.UTF-8 UTF-8" > /etc/locale.gen && echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && locale-gen +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV LC_CTYPE=C.UTF-8 +ENV LC_COLLATE=C.UTF-8 +ENV LOCALE_ARCHIVE /usr/lib/locale/locale-archive +CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 20ad069e2..e22cbe67b 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,7 +6,7 @@ - name: Install Postgres from nix binary cache become: yes shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#psql_15/bin" + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" #TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix @@ -22,6 +22,18 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supabase-groonga" when: stage2_nix +- name: Install debug symbols for postgres version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" + when: stage2_nix + +- name: Install source files for postgresql version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" + when: stage2_nix + - name: Set ownership and permissions for /etc/ssl/private become: yes file: diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..f78b56c97 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,9 +2,17 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -postgresql_major: "15" -postgresql_release: "15.1" -postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +# postgresql_major: "15" +# postgresql_release: "15.1" +# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c +postgres_major: + - "15" + - "16" + +# Full version strings for each major version +postgres_release: + postgres15: "15.8.1.003-staging-5" + postgres16: "16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl deleted file mode 100644 index 8419992a8..000000000 --- a/common-nix.vars.pkr.hcl +++ /dev/null @@ -1 +0,0 @@ -postgres-version = "15.8.1.003" diff --git a/flake.nix b/flake.nix index 3bb6d1a45..40202d2f0 100644 --- a/flake.nix +++ b/flake.nix @@ -318,6 +318,7 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; + postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; diff --git a/migrations/Dockerfile.dbmate b/migrations/Dockerfile.dbmate new file mode 100644 index 000000000..0af8193f4 --- /dev/null +++ b/migrations/Dockerfile.dbmate @@ -0,0 +1,23 @@ +FROM debian:bullseye-slim + +RUN apt-get update && apt-get install -y curl wget gnupg2 lsb-release + +RUN ARCH=$(dpkg --print-architecture); \ + case ${ARCH} in \ + amd64) DBMATE_ARCH="linux-amd64" ;; \ + arm64) DBMATE_ARCH="linux-arm64" ;; \ + *) echo "Unsupported architecture: ${ARCH}"; exit 1 ;; \ + esac && \ + curl -fsSL -o /usr/local/bin/dbmate \ + https://github.com/amacneil/dbmate/releases/latest/download/dbmate-${DBMATE_ARCH} && \ + chmod +x /usr/local/bin/dbmate + +RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list +RUN apt-get update && apt-get install -y postgresql-client-%VERSION% + +ENV PATH="/usr/lib/postgresql/%VERSION%/bin:${PATH}" + +RUN dbmate --version + +ENTRYPOINT ["dbmate"] \ No newline at end of file diff --git a/migrations/docker-compose.yaml b/migrations/docker-compose.yaml index 2609d6008..1b3e8b143 100644 --- a/migrations/docker-compose.yaml +++ b/migrations/docker-compose.yaml @@ -33,7 +33,9 @@ services: command: pg_prove /tests/test.sql dbmate: - image: amacneil/dbmate:1.16.2 + build: + context: . + dockerfile: Dockerfile.dbmate depends_on: db: condition: service_healthy diff --git a/migrations/schema-15.sql b/migrations/schema-15.sql new file mode 100644 index 000000000..1bff8b9d8 --- /dev/null +++ b/migrations/schema-15.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT secrets.id, + secrets.name, + secrets.description, + secrets.secret, + CASE + WHEN (secrets.secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (secrets.key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secrets.secret, 'base64'::text), convert_to(((((secrets.id)::text || secrets.description) || (secrets.created_at)::text) || (secrets.updated_at)::text), 'utf8'::name), secrets.key_id, secrets.nonce), 'utf8'::name) + END + END AS decrypted_secret, + secrets.key_id, + secrets.nonce, + secrets.created_at, + secrets.updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- + diff --git a/migrations/schema-16.sql b/migrations/schema-16.sql new file mode 100644 index 000000000..49873a109 --- /dev/null +++ b/migrations/schema-16.sql @@ -0,0 +1,1064 @@ +SET statement_timeout = 0; +SET lock_timeout = 0; +SET idle_in_transaction_session_timeout = 0; +SET client_encoding = 'UTF8'; +SET standard_conforming_strings = on; +SELECT pg_catalog.set_config('search_path', '', false); +SET check_function_bodies = false; +SET xmloption = content; +SET client_min_messages = warning; +SET row_security = off; + +-- +-- Name: auth; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA auth; + + +-- +-- Name: extensions; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA extensions; + + +-- +-- Name: graphql; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql; + + +-- +-- Name: graphql_public; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA graphql_public; + + +-- +-- Name: pgbouncer; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgbouncer; + + +-- +-- Name: pgsodium; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA pgsodium; + + +-- +-- Name: pgsodium; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgsodium WITH SCHEMA pgsodium; + + +-- +-- Name: EXTENSION pgsodium; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgsodium IS 'Pgsodium is a modern cryptography library for Postgres.'; + + +-- +-- Name: realtime; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA realtime; + + +-- +-- Name: storage; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA storage; + + +-- +-- Name: vault; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA vault; + + +-- +-- Name: pg_graphql; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_graphql WITH SCHEMA graphql; + + +-- +-- Name: EXTENSION pg_graphql; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_graphql IS 'pg_graphql: GraphQL support'; + + +-- +-- Name: pg_stat_statements; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pg_stat_statements WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pg_stat_statements; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pg_stat_statements IS 'track planning and execution statistics of all SQL statements executed'; + + +-- +-- Name: pgcrypto; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgcrypto WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgcrypto; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgcrypto IS 'cryptographic functions'; + + +-- +-- Name: pgjwt; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS pgjwt WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION pgjwt; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION pgjwt IS 'JSON Web Token API for Postgresql'; + + +-- +-- Name: supabase_vault; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS supabase_vault WITH SCHEMA vault; + + +-- +-- Name: EXTENSION supabase_vault; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION supabase_vault IS 'Supabase Vault Extension'; + + +-- +-- Name: uuid-ossp; Type: EXTENSION; Schema: -; Owner: - +-- + +CREATE EXTENSION IF NOT EXISTS "uuid-ossp" WITH SCHEMA extensions; + + +-- +-- Name: EXTENSION "uuid-ossp"; Type: COMMENT; Schema: -; Owner: - +-- + +COMMENT ON EXTENSION "uuid-ossp" IS 'generate universally unique identifiers (UUIDs)'; + + +-- +-- Name: email(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.email() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.email', true), '')::text; +$$; + + +-- +-- Name: role(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.role() RETURNS text + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.role', true), '')::text; +$$; + + +-- +-- Name: uid(); Type: FUNCTION; Schema: auth; Owner: - +-- + +CREATE FUNCTION auth.uid() RETURNS uuid + LANGUAGE sql STABLE + AS $$ + select nullif(current_setting('request.jwt.claim.sub', true), '')::uuid; +$$; + + +-- +-- Name: grant_pg_cron_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_cron_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_cron' + ) + THEN + grant usage on schema cron to postgres with grant option; + + alter default privileges in schema cron grant all on tables to postgres with grant option; + alter default privileges in schema cron grant all on functions to postgres with grant option; + alter default privileges in schema cron grant all on sequences to postgres with grant option; + + alter default privileges for user supabase_admin in schema cron grant all + on sequences to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on tables to postgres with grant option; + alter default privileges for user supabase_admin in schema cron grant all + on functions to postgres with grant option; + + grant all privileges on all tables in schema cron to postgres with grant option; + revoke all on table cron.job from postgres; + grant select on table cron.job to postgres with grant option; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_cron_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_cron_access() IS 'Grants access to pg_cron'; + + +-- +-- Name: grant_pg_graphql_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_graphql_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ +DECLARE + func_is_graphql_resolve bool; +BEGIN + func_is_graphql_resolve = ( + SELECT n.proname = 'resolve' + FROM pg_event_trigger_ddl_commands() AS ev + LEFT JOIN pg_catalog.pg_proc AS n + ON ev.objid = n.oid + ); + + IF func_is_graphql_resolve + THEN + -- Update public wrapper to pass all arguments through to the pg_graphql resolve func + DROP FUNCTION IF EXISTS graphql_public.graphql; + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language sql + as $$ + select graphql.resolve( + query := query, + variables := coalesce(variables, '{}'), + "operationName" := "operationName", + extensions := extensions + ); + $$; + + -- This hook executes when `graphql.resolve` is created. That is not necessarily the last + -- function in the extension so we need to grant permissions on existing entities AND + -- update default permissions to any others that are created after `graphql.resolve` + grant usage on schema graphql to postgres, anon, authenticated, service_role; + grant select on all tables in schema graphql to postgres, anon, authenticated, service_role; + grant execute on all functions in schema graphql to postgres, anon, authenticated, service_role; + grant all on all sequences in schema graphql to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on tables to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on functions to postgres, anon, authenticated, service_role; + alter default privileges in schema graphql grant all on sequences to postgres, anon, authenticated, service_role; + + -- Allow postgres role to allow granting usage on graphql and graphql_public schemas to custom roles + grant usage on schema graphql_public to postgres with grant option; + grant usage on schema graphql to postgres with grant option; + END IF; + +END; +$_$; + + +-- +-- Name: FUNCTION grant_pg_graphql_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_graphql_access() IS 'Grants access to pg_graphql'; + + +-- +-- Name: grant_pg_net_access(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.grant_pg_net_access() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_event_trigger_ddl_commands() AS ev + JOIN pg_extension AS ext + ON ev.objid = ext.oid + WHERE ext.extname = 'pg_net' + ) + THEN + IF NOT EXISTS ( + SELECT 1 + FROM pg_roles + WHERE rolname = 'supabase_functions_admin' + ) + THEN + CREATE USER supabase_functions_admin NOINHERIT CREATEROLE LOGIN NOREPLICATION; + END IF; + + GRANT USAGE ON SCHEMA net TO supabase_functions_admin, postgres, anon, authenticated, service_role; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SECURITY DEFINER; + + ALTER function net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + ALTER function net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) SET search_path = net; + + REVOKE ALL ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + REVOKE ALL ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) FROM PUBLIC; + + GRANT EXECUTE ON FUNCTION net.http_get(url text, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + GRANT EXECUTE ON FUNCTION net.http_post(url text, body jsonb, params jsonb, headers jsonb, timeout_milliseconds integer) TO supabase_functions_admin, postgres, anon, authenticated, service_role; + END IF; +END; +$$; + + +-- +-- Name: FUNCTION grant_pg_net_access(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.grant_pg_net_access() IS 'Grants access to pg_net'; + + +-- +-- Name: pgrst_ddl_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_ddl_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + cmd record; +BEGIN + FOR cmd IN SELECT * FROM pg_event_trigger_ddl_commands() + LOOP + IF cmd.command_tag IN ( + 'CREATE SCHEMA', 'ALTER SCHEMA' + , 'CREATE TABLE', 'CREATE TABLE AS', 'SELECT INTO', 'ALTER TABLE' + , 'CREATE FOREIGN TABLE', 'ALTER FOREIGN TABLE' + , 'CREATE VIEW', 'ALTER VIEW' + , 'CREATE MATERIALIZED VIEW', 'ALTER MATERIALIZED VIEW' + , 'CREATE FUNCTION', 'ALTER FUNCTION' + , 'CREATE TRIGGER' + , 'CREATE TYPE', 'ALTER TYPE' + , 'CREATE RULE' + , 'COMMENT' + ) + -- don't notify in case of CREATE TEMP table or other objects created on pg_temp + AND cmd.schema_name is distinct from 'pg_temp' + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: pgrst_drop_watch(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.pgrst_drop_watch() RETURNS event_trigger + LANGUAGE plpgsql + AS $$ +DECLARE + obj record; +BEGIN + FOR obj IN SELECT * FROM pg_event_trigger_dropped_objects() + LOOP + IF obj.object_type IN ( + 'schema' + , 'table' + , 'foreign table' + , 'view' + , 'materialized view' + , 'function' + , 'trigger' + , 'type' + , 'rule' + ) + AND obj.is_temporary IS false -- no pg_temp objects + THEN + NOTIFY pgrst, 'reload schema'; + END IF; + END LOOP; +END; $$; + + +-- +-- Name: set_graphql_placeholder(); Type: FUNCTION; Schema: extensions; Owner: - +-- + +CREATE FUNCTION extensions.set_graphql_placeholder() RETURNS event_trigger + LANGUAGE plpgsql + AS $_$ + DECLARE + graphql_is_dropped bool; + BEGIN + graphql_is_dropped = ( + SELECT ev.schema_name = 'graphql_public' + FROM pg_event_trigger_dropped_objects() AS ev + WHERE ev.schema_name = 'graphql_public' + ); + + IF graphql_is_dropped + THEN + create or replace function graphql_public.graphql( + "operationName" text default null, + query text default null, + variables jsonb default null, + extensions jsonb default null + ) + returns jsonb + language plpgsql + as $$ + DECLARE + server_version float; + BEGIN + server_version = (SELECT (SPLIT_PART((select version()), ' ', 2))::float); + + IF server_version >= 14 THEN + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql extension is not enabled.' + ) + ) + ); + ELSE + RETURN jsonb_build_object( + 'errors', jsonb_build_array( + jsonb_build_object( + 'message', 'pg_graphql is only available on projects running Postgres 14 onwards.' + ) + ) + ); + END IF; + END; + $$; + END IF; + + END; +$_$; + + +-- +-- Name: FUNCTION set_graphql_placeholder(); Type: COMMENT; Schema: extensions; Owner: - +-- + +COMMENT ON FUNCTION extensions.set_graphql_placeholder() IS 'Reintroduces placeholder function for graphql_public.graphql'; + + +-- +-- Name: get_auth(text); Type: FUNCTION; Schema: pgbouncer; Owner: - +-- + +CREATE FUNCTION pgbouncer.get_auth(p_usename text) RETURNS TABLE(username text, password text) + LANGUAGE plpgsql SECURITY DEFINER + AS $$ +BEGIN + RAISE WARNING 'PgBouncer auth request: %', p_usename; + + RETURN QUERY + SELECT usename::TEXT, passwd::TEXT FROM pg_catalog.pg_shadow + WHERE usename = p_usename; +END; +$$; + + +-- +-- Name: extension(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.extension(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +_filename text; +BEGIN + select string_to_array(name, '/') into _parts; + select _parts[array_length(_parts,1)] into _filename; + -- @todo return the last part instead of 2 + return split_part(_filename, '.', 2); +END +$$; + + +-- +-- Name: filename(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.filename(name text) RETURNS text + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[array_length(_parts,1)]; +END +$$; + + +-- +-- Name: foldername(text); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.foldername(name text) RETURNS text[] + LANGUAGE plpgsql + AS $$ +DECLARE +_parts text[]; +BEGIN + select string_to_array(name, '/') into _parts; + return _parts[1:array_length(_parts,1)-1]; +END +$$; + + +-- +-- Name: search(text, text, integer, integer, integer); Type: FUNCTION; Schema: storage; Owner: - +-- + +CREATE FUNCTION storage.search(prefix text, bucketname text, limits integer DEFAULT 100, levels integer DEFAULT 1, offsets integer DEFAULT 0) RETURNS TABLE(name text, id uuid, updated_at timestamp with time zone, created_at timestamp with time zone, last_accessed_at timestamp with time zone, metadata jsonb) + LANGUAGE plpgsql + AS $$ +DECLARE +_bucketId text; +BEGIN + -- will be replaced by migrations when server starts + -- saving space for cloud-init +END +$$; + + +-- +-- Name: secrets_encrypt_secret_secret(); Type: FUNCTION; Schema: vault; Owner: - +-- + +CREATE FUNCTION vault.secrets_encrypt_secret_secret() RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + new.secret = CASE WHEN new.secret IS NULL THEN NULL ELSE + CASE WHEN new.key_id IS NULL THEN NULL ELSE pg_catalog.encode( + pgsodium.crypto_aead_det_encrypt( + pg_catalog.convert_to(new.secret, 'utf8'), + pg_catalog.convert_to((new.id::text || new.description::text || new.created_at::text || new.updated_at::text)::text, 'utf8'), + new.key_id::uuid, + new.nonce + ), + 'base64') END END; + RETURN new; + END; + $$; + + +SET default_tablespace = ''; + +SET default_table_access_method = heap; + +-- +-- Name: audit_log_entries; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.audit_log_entries ( + instance_id uuid, + id uuid NOT NULL, + payload json, + created_at timestamp with time zone +); + + +-- +-- Name: TABLE audit_log_entries; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.audit_log_entries IS 'Auth: Audit trail for user actions.'; + + +-- +-- Name: instances; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.instances ( + id uuid NOT NULL, + uuid uuid, + raw_base_config text, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE instances; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.instances IS 'Auth: Manages users across multiple sites.'; + + +-- +-- Name: refresh_tokens; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.refresh_tokens ( + instance_id uuid, + id bigint NOT NULL, + token character varying(255), + user_id character varying(255), + revoked boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE refresh_tokens; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.refresh_tokens IS 'Auth: Store of tokens used to refresh JWT tokens once they expire.'; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE; Schema: auth; Owner: - +-- + +CREATE SEQUENCE auth.refresh_tokens_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: refresh_tokens_id_seq; Type: SEQUENCE OWNED BY; Schema: auth; Owner: - +-- + +ALTER SEQUENCE auth.refresh_tokens_id_seq OWNED BY auth.refresh_tokens.id; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.schema_migrations ( + version character varying(255) NOT NULL +); + + +-- +-- Name: TABLE schema_migrations; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.schema_migrations IS 'Auth: Manages updates to the auth system.'; + + +-- +-- Name: users; Type: TABLE; Schema: auth; Owner: - +-- + +CREATE TABLE auth.users ( + instance_id uuid, + id uuid NOT NULL, + aud character varying(255), + role character varying(255), + email character varying(255), + encrypted_password character varying(255), + confirmed_at timestamp with time zone, + invited_at timestamp with time zone, + confirmation_token character varying(255), + confirmation_sent_at timestamp with time zone, + recovery_token character varying(255), + recovery_sent_at timestamp with time zone, + email_change_token character varying(255), + email_change character varying(255), + email_change_sent_at timestamp with time zone, + last_sign_in_at timestamp with time zone, + raw_app_meta_data jsonb, + raw_user_meta_data jsonb, + is_super_admin boolean, + created_at timestamp with time zone, + updated_at timestamp with time zone +); + + +-- +-- Name: TABLE users; Type: COMMENT; Schema: auth; Owner: - +-- + +COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure schema.'; + + +-- +-- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.schema_migrations ( + version character varying(128) NOT NULL +); + + +-- +-- Name: buckets; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.buckets ( + id text NOT NULL, + name text NOT NULL, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now() +); + + +-- +-- Name: migrations; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.migrations ( + id integer NOT NULL, + name character varying(100) NOT NULL, + hash character varying(40) NOT NULL, + executed_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP +); + + +-- +-- Name: objects; Type: TABLE; Schema: storage; Owner: - +-- + +CREATE TABLE storage.objects ( + id uuid DEFAULT extensions.uuid_generate_v4() NOT NULL, + bucket_id text, + name text, + owner uuid, + created_at timestamp with time zone DEFAULT now(), + updated_at timestamp with time zone DEFAULT now(), + last_accessed_at timestamp with time zone DEFAULT now(), + metadata jsonb +); + + +-- +-- Name: decrypted_secrets; Type: VIEW; Schema: vault; Owner: - +-- + +CREATE VIEW vault.decrypted_secrets AS + SELECT id, + name, + description, + secret, + CASE + WHEN (secret IS NULL) THEN NULL::text + ELSE + CASE + WHEN (key_id IS NULL) THEN NULL::text + ELSE convert_from(pgsodium.crypto_aead_det_decrypt(decode(secret, 'base64'::text), convert_to(((((id)::text || description) || (created_at)::text) || (updated_at)::text), 'utf8'::name), key_id, nonce), 'utf8'::name) + END + END AS decrypted_secret, + key_id, + nonce, + created_at, + updated_at + FROM vault.secrets; + + +-- +-- Name: refresh_tokens id; Type: DEFAULT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens ALTER COLUMN id SET DEFAULT nextval('auth.refresh_tokens_id_seq'::regclass); + + +-- +-- Name: audit_log_entries audit_log_entries_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.audit_log_entries + ADD CONSTRAINT audit_log_entries_pkey PRIMARY KEY (id); + + +-- +-- Name: instances instances_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.instances + ADD CONSTRAINT instances_pkey PRIMARY KEY (id); + + +-- +-- Name: refresh_tokens refresh_tokens_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.refresh_tokens + ADD CONSTRAINT refresh_tokens_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: users users_email_key; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_email_key UNIQUE (email); + + +-- +-- Name: users users_pkey; Type: CONSTRAINT; Schema: auth; Owner: - +-- + +ALTER TABLE ONLY auth.users + ADD CONSTRAINT users_pkey PRIMARY KEY (id); + + +-- +-- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); + + +-- +-- Name: buckets buckets_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_pkey PRIMARY KEY (id); + + +-- +-- Name: migrations migrations_name_key; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_name_key UNIQUE (name); + + +-- +-- Name: migrations migrations_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.migrations + ADD CONSTRAINT migrations_pkey PRIMARY KEY (id); + + +-- +-- Name: objects objects_pkey; Type: CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_pkey PRIMARY KEY (id); + + +-- +-- Name: audit_logs_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX audit_logs_instance_id_idx ON auth.audit_log_entries USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_idx ON auth.refresh_tokens USING btree (instance_id); + + +-- +-- Name: refresh_tokens_instance_id_user_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_instance_id_user_id_idx ON auth.refresh_tokens USING btree (instance_id, user_id); + + +-- +-- Name: refresh_tokens_token_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX refresh_tokens_token_idx ON auth.refresh_tokens USING btree (token); + + +-- +-- Name: users_instance_id_email_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_email_idx ON auth.users USING btree (instance_id, email); + + +-- +-- Name: users_instance_id_idx; Type: INDEX; Schema: auth; Owner: - +-- + +CREATE INDEX users_instance_id_idx ON auth.users USING btree (instance_id); + + +-- +-- Name: bname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bname ON storage.buckets USING btree (name); + + +-- +-- Name: bucketid_objname; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE UNIQUE INDEX bucketid_objname ON storage.objects USING btree (bucket_id, name); + + +-- +-- Name: name_prefix_search; Type: INDEX; Schema: storage; Owner: - +-- + +CREATE INDEX name_prefix_search ON storage.objects USING btree (name text_pattern_ops); + + +-- +-- Name: buckets buckets_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.buckets + ADD CONSTRAINT buckets_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects objects_bucketId_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT "objects_bucketId_fkey" FOREIGN KEY (bucket_id) REFERENCES storage.buckets(id); + + +-- +-- Name: objects objects_owner_fkey; Type: FK CONSTRAINT; Schema: storage; Owner: - +-- + +ALTER TABLE ONLY storage.objects + ADD CONSTRAINT objects_owner_fkey FOREIGN KEY (owner) REFERENCES auth.users(id); + + +-- +-- Name: objects; Type: ROW SECURITY; Schema: storage; Owner: - +-- + +ALTER TABLE storage.objects ENABLE ROW LEVEL SECURITY; + +-- +-- Name: supabase_realtime; Type: PUBLICATION; Schema: -; Owner: - +-- + +CREATE PUBLICATION supabase_realtime WITH (publish = 'insert, update, delete, truncate'); + + +-- +-- Name: issue_graphql_placeholder; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_graphql_placeholder ON sql_drop + WHEN TAG IN ('DROP EXTENSION') + EXECUTE FUNCTION extensions.set_graphql_placeholder(); + + +-- +-- Name: issue_pg_cron_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_cron_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_cron_access(); + + +-- +-- Name: issue_pg_graphql_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_graphql_access ON ddl_command_end + WHEN TAG IN ('CREATE FUNCTION') + EXECUTE FUNCTION extensions.grant_pg_graphql_access(); + + +-- +-- Name: issue_pg_net_access; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER issue_pg_net_access ON ddl_command_end + WHEN TAG IN ('CREATE EXTENSION') + EXECUTE FUNCTION extensions.grant_pg_net_access(); + + +-- +-- Name: pgrst_ddl_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_ddl_watch ON ddl_command_end + EXECUTE FUNCTION extensions.pgrst_ddl_watch(); + + +-- +-- Name: pgrst_drop_watch; Type: EVENT TRIGGER; Schema: -; Owner: - +-- + +CREATE EVENT TRIGGER pgrst_drop_watch ON sql_drop + EXECUTE FUNCTION extensions.pgrst_drop_watch(); + + +-- +-- PostgreSQL database dump complete +-- + + +-- +-- Dbmate schema migrations +-- + diff --git a/migrations/schema.sql b/migrations/schema.sql index f68d131e8..1bff8b9d8 100644 --- a/migrations/schema.sql +++ b/migrations/schema.sql @@ -737,7 +737,7 @@ COMMENT ON TABLE auth.users IS 'Auth: Stores user login data within a secure sch -- CREATE TABLE public.schema_migrations ( - version character varying(255) NOT NULL + version character varying(128) NOT NULL ); diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..c4776b3a8 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -25,16 +25,23 @@ function install_nix() { function execute_stage2_playbook { + echo "POSTGRES_MAJOR_VERSION: ${POSTGRES_MAJOR_VERSION}" + echo "GIT_SHA: ${GIT_SHA}" sudo tee /etc/ansible/ansible.cfg < Date: Thu, 10 Oct 2024 13:10:37 -0400 Subject: [PATCH 007/109] chore: rm dead code --- flake.nix | 2 -- 1 file changed, 2 deletions(-) diff --git a/flake.nix b/flake.nix index 40202d2f0..1e4fa8132 100644 --- a/flake.nix +++ b/flake.nix @@ -141,8 +141,6 @@ #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; - #we will add supported versions to this list in the future - supportedVersions = [ "15" ]; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL From 3454b624004a1f2a3f8823941768af3412357c87 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 12:04:50 -0400 Subject: [PATCH 008/109] chore: resolve versions from ansible/vars.yml instead of packer file create a matrix build for 15 16 (or how ever many versions there are) --- ...ublish-nix-pgupgrade-bin-flake-version.yml | 36 ++++++++++++---- .../publish-nix-pgupgrade-scripts.yml | 41 +++++++++++++++---- 2 files changed, 60 insertions(+), 17 deletions(-) diff --git a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml index 5b985f4be..ebfa27fe3 100644 --- a/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml +++ b/.github/workflows/publish-nix-pgupgrade-bin-flake-version.yml @@ -11,8 +11,28 @@ permissions: id-token: write jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + publish-staging: + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo @@ -21,10 +41,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" @@ -58,6 +76,10 @@ jobs: publish-prod: runs-on: ubuntu-latest if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + needs: prepare + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo @@ -66,10 +88,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" echo "major_version=$(echo $VERSION | cut -d'.' -f1)" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index eb5f7a755..59de9a788 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -7,7 +7,6 @@ on: - release/* paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' - - 'common-nix.vars.pkr.hcl' workflow_dispatch: inputs: postgresVersion: @@ -18,20 +17,40 @@ permissions: id-token: write jobs: + prepare: + runs-on: ubuntu-latest + outputs: + postgres_versions: ${{ steps.set-versions.outputs.postgres_versions }} + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL versions + id: set-versions + run: | + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT + publish-staging: + needs: prepare runs-on: ubuntu-latest + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} steps: - name: Checkout Repo uses: actions/checkout@v3 + - uses: DeterminateSystems/nix-installer-action@main + - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts @@ -61,9 +80,15 @@ jobs: SLACK_FOOTER: '' publish-prod: + needs: prepare runs-on: ubuntu-latest if: github.ref_name == 'develop' || contains( github.ref, 'release' ) + strategy: + matrix: + postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} + + steps: - name: Checkout Repo uses: actions/checkout@v3 @@ -71,10 +96,8 @@ jobs: - name: Grab release version id: process_release_version run: | - VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') - if [[ "${{ inputs.postgresVersion }}" != "" ]]; then - VERSION=${{ inputs.postgresVersion }} - fi + VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts From d576b0174bb7d7dacf9761b901b5e668904baaea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 13:15:47 -0400 Subject: [PATCH 009/109] feat: WIP orioledb 16 --- flake.nix | 6 +++--- nix/ext/orioledb.nix | 6 +++--- nix/overlays/psql_16-oriole.nix | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 1e4fa8132..74b98807b 100644 --- a/flake.nix +++ b/flake.nix @@ -37,7 +37,7 @@ # pull them from the overlays/ directory automatically, but we don't # want to have an arbitrary order, since it might matter. being # explicit is better. - (import ./nix/overlays/cargo-pgrx.nix) + (import ./nix/overlays/cargo-pgrx-0-11-3.nix) (import ./nix/overlays/psql_16-oriole.nix) ]; @@ -289,7 +289,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - # psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + psql_orioledb_16 = makeOrioleDbPostgres "16_29" postgresql_orioledb_16; }; # Find the active PostgreSQL version @@ -311,7 +311,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - #psql_orioledb_16 = makeOrioleDbPostgres "16_23" postgresql_orioledb_16; + psql_orioledb_16 = postgresVersions.psql_orioledb_16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 4d8c51bbd..a0ffc119c 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -7,11 +7,11 @@ stdenv.mkDerivation rec { owner = "orioledb"; repo = "orioledb"; rev = "main"; - sha256 = "sha256-QbDp9S8JXO66sfaHZIQ3wFCVRxsAaaNSRgC6hvL3EKY="; + sha256 = "sha256-VWjb2JHYad0VZkId70m8UOhRTJRGY4nkEuC7m5ae7w4="; }; - version = "patches16_23"; + version = "patches16_29"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=23"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=29"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index d55af10a2..00b1d4571 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_23"; + version = "16_29"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_23.tar.gz"; - sha256 = "sha256-xWmcqn3DYyBG0FsBNqPWTFzUidSJZgoPWI6Rt0N9oJ4="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_29.tar.gz"; + sha256 = "sha256-JiC4upjta7OWOfnuxgo/KL41tWcsBUyykOJakn6XjvA="; }; buildInputs = old.buildInputs ++ [ prev.bison From ce435309f714634b8b7a895fbe569a3414f89128 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 15 Oct 2024 13:24:50 -0400 Subject: [PATCH 010/109] feat: intro into CI building psql16 bundle oriole version --- docker/nix/build_nix.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 3cf8505c0..dfd769382 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -7,7 +7,10 @@ if [ -d "/workspace" ]; then fi nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link +#no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 +nix build .#psql_orioledb_16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 \ No newline at end of file From b6fdd1172e7c9dfda5e4b0a8094955e4ff60e1a9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 16 Oct 2024 09:50:15 -0400 Subject: [PATCH 011/109] test: working through build compat for oriole --- nix/ext/orioledb.nix | 8 ++++---- nix/overlays/psql_16-oriole.nix | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index a0ffc119c..266f552d8 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,12 +6,12 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "main"; - sha256 = "sha256-VWjb2JHYad0VZkId70m8UOhRTJRGY4nkEuC7m5ae7w4="; + rev = "0dafcb1bc799e9af393094c122c1c3c630797222"; + sha256 = "sha256-dsfDqUXkMeAkUI5l9+J09tsRZOGJVsqcKEVo5YAzMjU="; }; - version = "patches16_29"; + version = "patches16_30"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=29"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=30"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index 00b1d4571..d0eaa606c 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_29"; + version = "16_30"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_29.tar.gz"; - sha256 = "sha256-JiC4upjta7OWOfnuxgo/KL41tWcsBUyykOJakn6XjvA="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_30.tar.gz"; + sha256 = "sha256-VITYbOZYKgbtstdfG2wl7+wV+tl/DDYB34BkIM/Xnvs="; }; buildInputs = old.buildInputs ++ [ prev.bison From 859cbca7c60643f78b6aaaec1fdbd1d47fe1eb83 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 10:00:32 -0400 Subject: [PATCH 012/109] working orioledb build --- flake.nix | 4 +++- nix/ext/orioledb.nix | 10 +++++----- nix/overlays/psql_16-oriole.nix | 6 +++--- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/flake.nix b/flake.nix index 74b98807b..8875c726e 100644 --- a/flake.nix +++ b/flake.nix @@ -134,7 +134,9 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option - orioledbExtension = ourExtensions ++ [ ./nix/ext/orioledb.nix ]; + #we're not using timescaledb in the orioledb version of supabase extensions + orioleFilteredExtensions = builtins.filter (x: x != ./nix/ext/timescaledb.nix) ourExtensions; + orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 266f552d8..ff7c12a96 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,17 +6,17 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "0dafcb1bc799e9af393094c122c1c3c630797222"; - sha256 = "sha256-dsfDqUXkMeAkUI5l9+J09tsRZOGJVsqcKEVo5YAzMjU="; + rev = "bd8e32d0ebaafd0ea3ec3074233b65167f3b6fb7"; + sha256 = "sha256-bzH1SgPZ6q90HpqRsECY2XQPghEcd2Hg4X55G43unNo="; }; - version = "patches16_30"; + version = "patches16_31"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=30"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=31"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index d0eaa606c..20051315d 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,10 +1,10 @@ final: prev: { postgresql_16 = prev.postgresql_16.overrideAttrs (old: { pname = "postgresql_16"; - version = "16_30"; + version = "16_31"; src = prev.fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_30.tar.gz"; - sha256 = "sha256-VITYbOZYKgbtstdfG2wl7+wV+tl/DDYB34BkIM/Xnvs="; + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; + sha256 = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; }; buildInputs = old.buildInputs ++ [ prev.bison From a4b4998dc50103ba82c713ac4587ff239a936eee Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 10:04:31 -0400 Subject: [PATCH 013/109] fix: update value --- flake.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 8875c726e..e76672f4d 100644 --- a/flake.nix +++ b/flake.nix @@ -291,7 +291,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_orioledb_16 = makeOrioleDbPostgres "16_29" postgresql_orioledb_16; + psql_orioledb_16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; }; # Find the active PostgreSQL version From 8c13a58e2b77232f7d4d0769f414de0e96a7377b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 13:24:10 -0400 Subject: [PATCH 014/109] chore: cut staging only release for oriole16 --- .github/workflows/ami-release-nix.yml | 5 +++-- ansible/vars.yml | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 643f26fc7..f7005a4b0 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,8 +3,9 @@ name: Release AMI Nix on: push: branches: - - develop - - release/* + #- develop + #- release/* + - sam/oriole16 paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' diff --git a/ansible/vars.yml b/ansible/vars.yml index f78b56c97..30cbf2dab 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -2,17 +2,16 @@ supabase_internal: true ebssurrogate_mode: true async_mode: true -# postgresql_major: "15" -# postgresql_release: "15.1" -# postgresql_release_checksum: sha256:ea2cf059a85882654b989acd07edc121833164a30340faee0d3615cf7058e66c postgres_major: - "15" - "16" + - "oriole-16" # Full version strings for each major version postgres_release: postgres15: "15.8.1.003-staging-5" postgres16: "16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 07117e3828e3cbc2ab62552f27139d75179186ba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 13:31:47 -0400 Subject: [PATCH 015/109] narrow down to just oriole staging ami here --- ansible/vars.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 30cbf2dab..1f161f82e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -3,14 +3,14 @@ ebssurrogate_mode: true async_mode: true postgres_major: - - "15" - - "16" + #- "15" + #- "16" - "oriole-16" # Full version strings for each major version postgres_release: - postgres15: "15.8.1.003-staging-5" - postgres16: "16.3.1.000-staging-5" + #postgres15: "15.8.1.003-staging-5" + #postgres16: "16.3.1.000-staging-5" postgresoriole-16: "oriole-16.3.1.000-staging-1" # Non Postgres Extensions From 2ea1e907f5c4ae72396a251a87ca65586fd36e7c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:06:06 -0400 Subject: [PATCH 016/109] fix: make sure name lines up with gh action convention --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9c26efbb0..108483071 100644 --- a/flake.nix +++ b/flake.nix @@ -291,7 +291,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_orioledb_16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; + psql_oriole-16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; }; # Find the active PostgreSQL version @@ -313,7 +313,7 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - psql_orioledb_16 = postgresVersions.psql_orioledb_16; + psql_oriole-16 = postgresVersions.psql_oriole-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; inherit postgresql_15 postgresql_16; From c7f68df1eabefd8cb70900fb59920cd9e5ac848a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:11:26 -0400 Subject: [PATCH 017/109] fix: get correct naming convention in here too for oriole bundle --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 822648bd7..78c843cd1 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,7 +10,7 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_orioledb_16/bin -o psql_orioledb_16 +nix build .#psql_oriole-16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 From 870f591aacaf089d7395a57e485749f1c92b157a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 14:17:16 -0400 Subject: [PATCH 018/109] chore: bump to cut release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 1f161f82e..fe6bbfdc7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-1" + postgresoriole-16: "oriole-16.3.1.000-staging-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" From d483a0786bee7266ac975863695613912246bee2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 15:24:18 -0400 Subject: [PATCH 019/109] fix: make sure src and debug builds available for oriole too --- flake.nix | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 108483071..27f46a11a 100644 --- a/flake.nix +++ b/flake.nix @@ -307,6 +307,7 @@ }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; + postgresql_oriole-16 = postgresql_orioledb_16; in postgresVersions //{ supabase-groonga = supabase-groonga; @@ -316,9 +317,10 @@ psql_oriole-16 = postgresVersions.psql_oriole-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16; + inherit postgresql_15 postgresql_16 postgresql_oriole-16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; + postgresql_oriole-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb_16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -363,6 +365,28 @@ platforms = platforms.all; }; }; + postgresql_oriole-16_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-16-src"; + version = postgresql_oriole-16.version; + + src = postgresql_16.src; + + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; From 5e69b003408e84ccbf3d5d6aa63c5c055f3ca353 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 15:28:36 -0400 Subject: [PATCH 020/109] chore: bump to staging release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index fe6bbfdc7..413704ad4 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-2" + postgresoriole-16: "oriole-16.3.1.000-staging-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From b62e770796de2ed009e863fd8ef520f1a23d3975 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 16:27:44 -0400 Subject: [PATCH 021/109] fix: when oriole16 rm timescale from supautils conf --- ansible/tasks/stage2-setup-postgres.yml | 16 ++++++++++++++++ ansible/vars.yml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index e22cbe67b..e6a2616f5 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -3,6 +3,22 @@ # shell: | # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task + +- name: Check psql_version and modify supautils.conf if necessary + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Remove 'timescaledb,' from supautils.conf + ansible.builtin.lineinfile: + path: /etc/postgresql-custom/supautils.conf + regexp: 'timescaledb,' + line: '' + state: absent + when: is_psql_oriole_16 and stage2_nix + become: yes + - name: Install Postgres from nix binary cache become: yes shell: | diff --git a/ansible/vars.yml b/ansible/vars.yml index 413704ad4..0b870393d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-3" + postgresoriole-16: "oriole-16.3.1.000-staging-4" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 889dc70f07da836291004e029af5bbda8fe6443f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 17 Oct 2024 21:03:40 -0400 Subject: [PATCH 022/109] fix: also remove from postgresql conf --- ansible/tasks/stage2-setup-postgres.yml | 10 +++++++++- ansible/vars.yml | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index e6a2616f5..a9c167499 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,7 +10,15 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from supautils.conf + - name: Remove 'timescaledb,' from supautils.conf if oriole-16 build + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf if oriole-16 build + regexp: 'timescaledb,' + line: '' + state: absent + when: is_psql_oriole_16 and stage2_nix + become: yes + - name: Remove 'timescaledb,' from postgresql.conf ansible.builtin.lineinfile: path: /etc/postgresql-custom/supautils.conf regexp: 'timescaledb,' diff --git a/ansible/vars.yml b/ansible/vars.yml index 0b870393d..5c9331f17 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-4" + postgresoriole-16: "oriole-16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" From a67d8e1de65eee5f4719219831690c50cf8ba0a7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 08:39:12 -0400 Subject: [PATCH 023/109] fix: find all instances and remove in this condition for tiemscaledb --- ansible/tasks/stage2-setup-postgres.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index a9c167499..5b4ee2a28 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,20 +10,14 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from supautils.conf if oriole-16 build - ansible.builtin.lineinfile: - path: /etc/postgresql/postgresql.conf if oriole-16 build - regexp: 'timescaledb,' - line: '' - state: absent + - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build + ansible.builtin.command: + cmd: sed -i 's/timescaledb,//g' /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Remove 'timescaledb,' from postgresql.conf - ansible.builtin.lineinfile: - path: /etc/postgresql-custom/supautils.conf - regexp: 'timescaledb,' - line: '' - state: absent + - name: Remove 'timescaledb,' from supautils.conf + ansible.builtin.command: + cmd: sed -i 's/timescaledb,//g' /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes @@ -31,7 +25,6 @@ become: yes shell: | sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{psql_version}}/bin" -#TODO (samrose) switch pg_prove sourcing to develop branch once PR is merged when: stage2_nix - name: Install pg_prove from nix binary cache From 67382a646447b26546657b0aa187b826e7bba6ac Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 09:28:33 -0400 Subject: [PATCH 024/109] fix: maybe broke pgsodium with too many spaces --- ansible/tasks/stage2-setup-postgres.yml | 4 ++-- ansible/vars.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 5b4ee2a28..485a068f6 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -12,12 +12,12 @@ - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/timescaledb,//g' /etc/postgresql/postgresql.conf + cmd: sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - name: Remove 'timescaledb,' from supautils.conf ansible.builtin.command: - cmd: sed -i 's/timescaledb,//g' /etc/postgresql-custom/supautils.conf + cmd: sed -i 's/ timescaledb,//g' /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 5c9331f17..2360c2fc7 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -11,7 +11,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-6" # Non Postgres Extensions pgbouncer_release: "1.19.0" From c08b661756562529bbdb1eea08321cde35b5ea5e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 10:24:58 -0400 Subject: [PATCH 025/109] fix: create a symlink for missing path --- .github/workflows/test.yml | 3 ++- ansible/tasks/stage2-setup-postgres.yml | 9 +++++++++ ansible/vars.yml | 3 ++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1457eb100..c58df7d15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,8 @@ on: push: branches: - develop - pull_request: + #TODO PR uncomment this line + #pull_request: workflow_dispatch: jobs: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 485a068f6..400af65ac 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -256,6 +256,15 @@ line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh' when: stage2_nix +#TODO PR may need to remove this. +- name: Create symbolic link for pgsodium_getkey script + file: + src: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" + dest: "/usr/lib/postgresql/share/postgresql/extension/pgsodium_getkey" + state: link + become: yes + when: stage2_nix + - name: Append GRN_PLUGINS_DIR to /etc/environment.d/postgresql.env ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env diff --git a/ansible/vars.yml b/ansible/vars.yml index 2360c2fc7..e924d826e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -8,10 +8,11 @@ postgres_major: - "oriole-16" # Full version strings for each major version +# TOD PR uncomment these lines postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-6" + postgresoriole-16: "oriole-16.3.1.000-staging-5" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 59b44b6364f0cd0191e7ce35a34cd0e748e9a258 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 11:49:53 -0400 Subject: [PATCH 026/109] fix: try a more universal way to deactivate pgsodium at this stage --- ansible/tasks/test-image.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index d6e8223f7..669bfec79 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -4,11 +4,22 @@ - libtap-parser-sourcehandler-pgtap-perl when: debpkg_mode +# - name: Temporarily disable PG Sodium references in config +# become: yes +# become_user: postgres +# shell: +# cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf +# when: debpkg_mode or stage2_nix + - name: Temporarily disable PG Sodium references in config become: yes become_user: postgres shell: - cmd: sed -i.bak -e "s/pg_net,\ pgsodium,\ timescaledb/pg_net,\ timescaledb/g" -e "s/pgsodium.getkey_script=/#pgsodium.getkey_script=/g" /etc/postgresql/postgresql.conf + cmd: > + sed -i.bak + -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' + -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/' + /etc/postgresql/postgresql.conf when: debpkg_mode or stage2_nix - name: Start Postgres Database to load all extensions. From f9613716fa8b64a3815ec31f4529ef908042c8f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 12:24:20 -0400 Subject: [PATCH 027/109] chore: when oriole16 rm timescale plv8 postgis pgvector pgrouting --- ansible/tasks/stage2-setup-postgres.yml | 14 +++++++++----- flake.nix | 10 +++++++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 400af65ac..0b4e3963d 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -10,17 +10,21 @@ set_fact: is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" - - name: Remove 'timescaledb,' from postgresql.conf if oriole-16 build + - name: Remove specified extensions from postgresql.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf + cmd: > + sed -i 's/ timescaledb,//g' + /etc/postgresql/postgresql.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Remove 'timescaledb,' from supautils.conf + + - name: Remove specified extensions from supautils.conf if oriole-16 build ansible.builtin.command: - cmd: sed -i 's/ timescaledb,//g' /etc/postgresql-custom/supautils.conf + cmd: > + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' + /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes - - name: Install Postgres from nix binary cache become: yes shell: | diff --git a/flake.nix b/flake.nix index 27f46a11a..3c5cb0fc1 100644 --- a/flake.nix +++ b/flake.nix @@ -135,7 +135,15 @@ #Where we import and build the orioledb extension, we add on our custom extensions # plus the orioledb option #we're not using timescaledb in the orioledb version of supabase extensions - orioleFilteredExtensions = builtins.filter (x: x != ./nix/ext/timescaledb.nix) ourExtensions; + orioleFilteredExtensions = builtins.filter ( + x: + x != ./nix/ext/timescaledb.nix && + x != ./nix/ext/pgvector.nix && + x != ./nix/ext/plv8.nix && + x != ./nix/ext/postgis.nix && + x != ./nix/ext/pgrouting.nix + ) ourExtensions; + orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql From a8f1a6be717a8f3412c30140143cc5e4ba54308c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 12:34:36 -0400 Subject: [PATCH 028/109] fix: more handling of oriole16 differences --- ansible/tasks/test-image.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 669bfec79..da77fb166 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -63,6 +63,21 @@ failed_when: retval.rc != 0 when: debpkg_mode or stage2_nix +- name: Check psql_version and modify migrations if orioe-16 + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Remove specified extensions from SQL file + ansible.builtin.command: + cmd: > + sed -i '/\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\)/d' + /tmp/migrations/tests/test.sql + when: is_psql_oriole_16 and stage2_nix + become: yes + + - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval From c35fa68409acc61017ca0c11c529e7699b4e646e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 13:35:31 -0400 Subject: [PATCH 029/109] chore more oriole16 handling --- ansible/tasks/stage2-setup-postgres.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 0b4e3963d..dbc61e5fd 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -4,7 +4,7 @@ # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task -- name: Check psql_version and modify supautils.conf if necessary +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary block: - name: Check if psql_version is psql_oriole-16 set_fact: @@ -25,6 +25,7 @@ /etc/postgresql-custom/supautils.conf when: is_psql_oriole_16 and stage2_nix become: yes + - name: Install Postgres from nix binary cache become: yes shell: | @@ -219,13 +220,19 @@ recurse: yes when: stage2_nix -- name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir - shell: > - sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ - sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ - && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" - become: yes - when: stage2_nix +- name: Check psql_version and run postgis linking if not oriole-16 + block: + - name: Check if psql_version is psql_oriole-16 + set_fact: + is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + + - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir + shell: > + sudo mkdir -p /usr/lib/postgresql/share/postgresql/contrib && \ + sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ + && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" + become: yes + when: stage2_nix and not is_psql_oriole_16 - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: From d1e06d473e2e205e62bf539d771ca58085afe3c1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 14:25:28 -0400 Subject: [PATCH 030/109] fix: refine sed regex --- ansible/tasks/test-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index da77fb166..c253be3ed 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -72,12 +72,12 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\)/d' - /tmp/migrations/tests/test.sql - when: is_psql_oriole_16 and stage2_nix + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/test.sql + when: is_psql_oriole_16 become: yes + - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql register: retval From 304aa68b4856cd1fdb341ef9251d885bb5c2594d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 15:03:42 -0400 Subject: [PATCH 031/109] fix: also cond rm actual files --- ansible/tasks/test-image.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index c253be3ed..65c03871a 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -76,6 +76,25 @@ when: is_psql_oriole_16 become: yes + - name: Remove specified extension files from extensions directory + ansible.builtin.find: + paths: /tmp/migrations/tests/extensions + patterns: + - '*timescaledb*.sql' + - '*postgis*.sql' + - '*pgrouting*.sql' + - '*plv8*.sql' + - '*pgvector*.sql' + register: files_to_remove + when: is_psql_oriole_16 + + - name: Delete matched extension files + ansible.builtin.file: + path: "{{ item.path }}" + state: absent + loop: "{{ files_to_remove.files }}" + when: is_psql_oriole_16 + become: yes - name: Run migrations tests From 55b97630abef9a8d807dbed02bb3b8f8277b1925 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 16:05:02 -0400 Subject: [PATCH 032/109] fix: correct dir --- ansible/tasks/test-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 65c03871a..f342d2e00 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -72,7 +72,7 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/test.sql + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql when: is_psql_oriole_16 become: yes From 292327efcc35f5563e41cd00328d68b956ff36b2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 18 Oct 2024 16:48:41 -0400 Subject: [PATCH 033/109] chore: bump to release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e924d826e..56fc36357 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-5" + postgresoriole-16: "oriole-16.3.1.000-staging-7" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 32163be55fc44bc5cbbdc8f829232232a9fc9fb0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 21 Oct 2024 15:44:09 -0400 Subject: [PATCH 034/109] feat: parse and handle oropledb-16 in start-server --- flake.nix | 1 + nix/tools/run-server.sh.in | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/flake.nix b/flake.nix index 3c5cb0fc1..940e7564c 100644 --- a/flake.nix +++ b/flake.nix @@ -454,6 +454,7 @@ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 977a437fb..1a634b8d6 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -1,7 +1,7 @@ #!/usr/bin/env bash # shellcheck shell=bash [ ! -z "$DEBUG" ] && set -x -# first argument should be '15' or '16' for the version + if [ "$1" == "15" ]; then echo "Starting server for PSQL 15" PSQL15=@PSQL15_BINDIR@ @@ -61,5 +61,17 @@ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ -e "\$a\\ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" + +remove_config_items() { + if [ "$1" == "orioledb-16" ]; then + # Remove items from supautils.conf + sed -i '/supautils.privileged_extensions/d' "$DATDIR/supautils.conf" + + # Remove items from postgresql.conf + sed -i 's/ timescaledb,//g' "$DATDIR/postgresql.conf" + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + fi +} +remove_config_items "$1" export GRN_PLUGINS_DIR=$GROONGA/lib/groonga/plugins postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp From e1c199c3a0e8a4d41c320fc6a1f715aa81f9c93d Mon Sep 17 00:00:00 2001 From: samrose Date: Mon, 21 Oct 2024 15:51:11 -0400 Subject: [PATCH 035/109] feat: start-client for oriole16 --- flake.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/flake.nix b/flake.nix index 940e7564c..af25775ad 100644 --- a/flake.nix +++ b/flake.nix @@ -484,6 +484,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ From 6a5648bb03ac96c1b420a930cd371187d8a17108 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 22 Oct 2024 08:56:28 -0400 Subject: [PATCH 036/109] wip for orioledb push to build on linux --- flake.nix | 104 ++++++++++++++++++++++++-------- nix/overlays/psql_16-oriole.nix | 6 +- 2 files changed, 83 insertions(+), 27 deletions(-) diff --git a/flake.nix b/flake.nix index af25775ad..71e671c67 100644 --- a/flake.nix +++ b/flake.nix @@ -116,7 +116,7 @@ ./nix/ext/pg_cron.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix - ./nix/ext/pg_net.nix + #./nix/ext/pg_net.nix ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix @@ -141,13 +141,14 @@ x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && - x != ./nix/ext/pgrouting.nix + x != ./nix/ext/pgrouting.nix && + x != ./nix/ext/pg_net.nix ) ourExtensions; orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; #this var is a convenience setting to import the orioledb patched version of postgresql - postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb_16; + postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb; #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: pkgs.postgresql."postgresql_${version}"; @@ -242,26 +243,81 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; - makeOrioleDbPostgresBin = version: patchedPostgres: - let - postgresql = patchedPostgres; - upstreamExts = map - (ext: { - name = postgresql.pkgs."${ext}".pname; - version = postgresql.pkgs."${ext}".version; - }) - orioledbPsqlExtensions; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); - - pgbin = postgresql.withPackages (ps: - (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) - ); - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - }; - + # makeOrioleDbPostgresBin = version: patchedPostgres: + # let + # postgresql = patchedPostgres; + # upstreamExts = map + # (ext: { + # name = postgresql.pkgs."${ext}".pname; + # version = postgresql.pkgs."${ext}".version; + # }) + # orioledbPsqlExtensions; + # ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); + + # pgbin = postgresql.withPackages (ps: + # (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) + # ); + # in + # pkgs.symlinkJoin { + # inherit (pgbin) name version; + # paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + # }; +makeOrioleDbPostgresBin = version: patchedPostgres: + let + postgresql = patchedPostgres; + upstreamExts = map + (ext: { + name = patchedPostgres.pkgs."${ext}".pname; + version = patchedPostgres.pkgs."${ext}".version; + }) + orioledbPsqlExtensions; + ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); + + # Create the scope and new self/super for package resolution + scope = { + inherit (patchedPostgres) jitSupport; + inherit (oriole_pkgs.llvmPackages) llvm; + postgresql = patchedPostgres; + inherit (postgresql) stdenv'; + }; + newSelf = self // scope; + newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; + + # Get the packages using the passthru pattern + selectedPackages = + (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ + (makeOurOrioleDbPostgresPkgs version postgresql); + + # Build the environment with the selected packages + pgbin = oriole_pkgs.buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = [ postgresql ] ++ selectedPackages; + buildInputs = [ oriole_pkgs.makeWrapper ]; + passthru = postgresql.passthru // { + inherit postgresql; + args = selectedPackages; + }; + + postBuild = '' + mkdir -p $out/bin + for bin in ${postgresql}/bin/*; do + ln -sf $bin $out/bin/ + done + + rm -f $out/bin/postgres + makeWrapper ${postgresql}/bin/postgres $out/bin/postgres \ + --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ + --prefix PATH : $out/bin \ + --prefix LD_LIBRARY_PATH : $out/lib + ''; + }// { + version = postgresql.version; + }; + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; + }; # Create an attribute set, containing all the relevant packages for a # PostgreSQL install, wrapped up with a bow on top. There are three @@ -299,7 +355,7 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_oriole-16 = makeOrioleDbPostgres "16_31" postgresql_orioledb_16; + psql_oriole-16 = makeOrioleDbPostgres "16" postgresql_orioledb_16; }; # Find the active PostgreSQL version diff --git a/nix/overlays/psql_16-oriole.nix b/nix/overlays/psql_16-oriole.nix index 20051315d..309129f1f 100644 --- a/nix/overlays/psql_16-oriole.nix +++ b/nix/overlays/psql_16-oriole.nix @@ -1,6 +1,6 @@ final: prev: { - postgresql_16 = prev.postgresql_16.overrideAttrs (old: { - pname = "postgresql_16"; + pg_orioledb = prev.postgresql_16.overrideAttrs (old: { + pname = "postgresql_orioledb"; version = "16_31"; src = prev.fetchurl { url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; @@ -17,5 +17,5 @@ final: prev: { prev.perl ]; }); - postgresql_orioledb_16 = final.postgresql_16; + postgresql_orioledb = final.pg_orioledb; } From 049010ae6d3f2345bed72ddd5af92b059e825b3b Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 22 Oct 2024 11:12:49 -0400 Subject: [PATCH 037/109] fix: running server --- flake.nix | 59 ++++++++++++++++++++++++++++++-------- nix/tools/run-server.sh.in | 5 +--- 2 files changed, 48 insertions(+), 16 deletions(-) diff --git a/flake.nix b/flake.nix index 71e671c67..11d5425ea 100644 --- a/flake.nix +++ b/flake.nix @@ -116,7 +116,7 @@ ./nix/ext/pg_cron.nix ./nix/ext/pgsql-http.nix ./nix/ext/pg_plan_filter.nix - #./nix/ext/pg_net.nix + ./nix/ext/pg_net.nix ./nix/ext/pg_hashids.nix ./nix/ext/pgsodium.nix ./nix/ext/pg_graphql.nix @@ -273,7 +273,6 @@ makeOrioleDbPostgresBin = version: patchedPostgres: orioledbPsqlExtensions; ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); - # Create the scope and new self/super for package resolution scope = { inherit (patchedPostgres) jitSupport; inherit (oriole_pkgs.llvmPackages) llvm; @@ -283,12 +282,10 @@ makeOrioleDbPostgresBin = version: patchedPostgres: newSelf = self // scope; newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; - # Get the packages using the passthru pattern selectedPackages = (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql); - # Build the environment with the selected packages pgbin = oriole_pkgs.buildEnv { name = "postgresql-and-plugins-${postgresql.version}"; paths = [ postgresql ] ++ selectedPackages; @@ -299,18 +296,56 @@ makeOrioleDbPostgresBin = version: patchedPostgres: }; postBuild = '' + # Ensure all necessary directories exist mkdir -p $out/bin + mkdir -p $out/lib/postgresql + mkdir -p $out/share/postgresql/extension + + # Create proper links for all binaries with environment for bin in ${postgresql}/bin/*; do - ln -sf $bin $out/bin/ + binary_name=$(basename $bin) + rm -f "$out/bin/$binary_name" + makeWrapper $bin $out/bin/$binary_name \ + --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ + --prefix PATH : $out/bin \ + --prefix LD_LIBRARY_PATH : "$out/lib:$out/lib/postgresql" \ + --prefix PKG_CONFIG_PATH : "$out/lib/pkgconfig" \ + --set PGDATA "/var/lib/postgresql/${postgresql.psqlSchema}" \ + --set PGHOST "/run/postgresql" + done + + # Ensure extension files are properly linked + for pkg in ${postgresql} ${toString selectedPackages}; do + # Link shared objects + if [ -d "$pkg/lib" ]; then + find "$pkg/lib" -type f -name '*.so' -exec ln -sf {} "$out/lib/postgresql/" \; + fi + + # Link extension control and SQL files + if [ -d "$pkg/share/postgresql/extension" ]; then + find "$pkg/share/postgresql/extension" -type f \( -name '*.control' -o -name '*.sql' \) \ + -exec ln -sf {} "$out/share/postgresql/extension/" \; + fi done - - rm -f $out/bin/postgres - makeWrapper ${postgresql}/bin/postgres $out/bin/postgres \ - --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ - --prefix PATH : $out/bin \ - --prefix LD_LIBRARY_PATH : $out/lib + + # Create pg_config if it doesn't exist + if [ ! -e "$out/bin/pg_config" ]; then + makeWrapper ${postgresql}/bin/pg_config $out/bin/pg_config \ + --set prefix "$out" \ + --set includedir "$out/include" \ + --set pkgincludedir "$out/include/postgresql" \ + --set includedir-server "$out/include/postgresql/server" \ + --set libdir "$out/lib" \ + --set pkglibdir "$out/lib/postgresql" \ + --set localedir "$out/share/locale" \ + --set mandir "$out/share/man" \ + --set sharedir "$out/share/postgresql" \ + --set sysconfdir "/etc/postgresql" \ + --set pgxs "$out/lib/postgresql/pgxs/src/makefiles/pgxs.mk" \ + --set configure "--enable-nls --with-openssl --with-libxml --with-libxslt" + fi ''; - }// { + } // { version = postgresql.version; }; in diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 1a634b8d6..a6b6ad7b1 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -64,11 +64,8 @@ session_preload_libraries = 'supautils'" \ remove_config_items() { if [ "$1" == "orioledb-16" ]; then - # Remove items from supautils.conf - sed -i '/supautils.privileged_extensions/d' "$DATDIR/supautils.conf" - # Remove items from postgresql.conf - sed -i 's/ timescaledb,//g' "$DATDIR/postgresql.conf" + sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" fi } From c2e2801e8a7c4e3a47a31e3345d13e197b885b05 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 23 Oct 2024 19:09:17 -0400 Subject: [PATCH 038/109] chore: consolidate nix code that handles building of postgres versions --- flake.nix | 196 +++++---------------------------- nix/postgresql/default.nix | 4 +- nix/postgresql/generic.nix | 21 +++- nix/postgresql/orioledb-16.nix | 20 ++++ 4 files changed, 62 insertions(+), 179 deletions(-) create mode 100644 nix/postgresql/orioledb-16.nix diff --git a/flake.nix b/flake.nix index 11d5425ea..5d25c3ef7 100644 --- a/flake.nix +++ b/flake.nix @@ -24,26 +24,6 @@ pgsqlSuperuser = "supabase_admin"; nix2img = nix2container.packages.${system}.nix2container; - # The 'oriole_pkgs' variable holds all the upstream packages in nixpkgs, which - # we can use to build our own images; it is the common name to refer to - # a copy of nixpkgs which contains all its packages. - # it also serves as a base for importing the orioldb/postgres overlay to - #build the orioledb postgres patched version of postgresql16 - oriole_pkgs = import nixpkgs { - config = { allowUnfree = true; }; - inherit system; - overlays = [ - # NOTE (aseipp): add any needed overlays here. in theory we could - # pull them from the overlays/ directory automatically, but we don't - # want to have an arbitrary order, since it might matter. being - # explicit is better. - (import ./nix/overlays/cargo-pgrx-0-11-3.nix) - (import ./nix/overlays/psql_16-oriole.nix) - - ]; - }; - #This variable works the same as 'oriole_pkgs' but builds using the upstream - #nixpkgs builds of postgresql 15 and 16 + the overlays listed below pkgs = import nixpkgs { config = { allowUnfree = true; @@ -59,11 +39,7 @@ # explicit is better. (final: prev: { postgresql = final.callPackage ./nix/postgresql/default.nix { - inherit (final) lib; - inherit (final) stdenv; - inherit (final) fetchurl; - inherit (final) makeWrapper; - inherit (final) callPackage; + inherit (final) lib stdenv fetchurl makeWrapper callPackage buildEnv newScope; }; }) (import ./nix/overlays/cargo-pgrx-0-11-3.nix) @@ -141,17 +117,15 @@ x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && - x != ./nix/ext/pgrouting.nix && - x != ./nix/ext/pg_net.nix + x != ./nix/ext/pgrouting.nix ) ourExtensions; - orioledbExtension = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; + orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - #this var is a convenience setting to import the orioledb patched version of postgresql - postgresql_orioledb_16 = oriole_pkgs.postgresql_orioledb; - #postgis_override = pkgs.postgis_override; getPostgresqlPackage = version: - pkgs.postgresql."postgresql_${version}"; + if version == "orioledb_16" + then pkgs.postgresql."postgresql_orioledb-16" + else pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -180,7 +154,7 @@ }; extensions = ourExts; - # NOTE (aseipp): this field can be used to do cache busting (e.g. + # NOTE this field can be used to do cache busting (e.g. # force a rebuild of the psql packages) but also to helpfully inform # tools what version of the schema is being used, for forwards and # backwards compatibility @@ -188,22 +162,18 @@ }; }; - makeOurOrioleDbPostgresPkgs = version: patchedPostgres: - let postgresql = patchedPostgres; - in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; + # makeOurOrioleDbPostgresPkgs = version: patchedPostgres: + # let postgresql = patchedPostgres; + # in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; makeOurPostgresPkgs = version: - let postgresql = getPostgresqlPackage version; - in map (path: pkgs.callPackage path { inherit postgresql; }) ourExtensions; + let + postgresql = getPostgresqlPackage version; + extensionsToUse = if version == "orioledb-16" + then orioledbExtensions + else ourExtensions; + in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; - # Create an attrset that contains all the extensions included in a server for the orioledb version of postgresql + extension. - makeOurOrioleDbPostgresPkgsSet = version: patchedPostgres: - (builtins.listToAttrs (map - (drv: - { name = drv.pname; value = drv; } - ) - (makeOurOrioleDbPostgresPkgs version patchedPostgres))) - // { recurseForDerivations = true; }; # Create an attrset that contains all the extensions included in a server. makeOurPostgresPkgsSet = version: @@ -243,117 +213,6 @@ paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; }; - # makeOrioleDbPostgresBin = version: patchedPostgres: - # let - # postgresql = patchedPostgres; - # upstreamExts = map - # (ext: { - # name = postgresql.pkgs."${ext}".pname; - # version = postgresql.pkgs."${ext}".version; - # }) - # orioledbPsqlExtensions; - # ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version postgresql); - - # pgbin = postgresql.withPackages (ps: - # (map (ext: ps."${ext}") orioledbPsqlExtensions) ++ (makeOurOrioleDbPostgresPkgs version postgresql) - # ); - # in - # pkgs.symlinkJoin { - # inherit (pgbin) name version; - # paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - # }; -makeOrioleDbPostgresBin = version: patchedPostgres: - let - postgresql = patchedPostgres; - upstreamExts = map - (ext: { - name = patchedPostgres.pkgs."${ext}".pname; - version = patchedPostgres.pkgs."${ext}".version; - }) - orioledbPsqlExtensions; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurOrioleDbPostgresPkgs version patchedPostgres); - - scope = { - inherit (patchedPostgres) jitSupport; - inherit (oriole_pkgs.llvmPackages) llvm; - postgresql = patchedPostgres; - inherit (postgresql) stdenv'; - }; - newSelf = self // scope; - newSuper = { callPackage = oriole_pkgs.newScope (scope // postgresql.pkgs); }; - - selectedPackages = - (map (ext: postgresql.pkgs."${ext}") orioledbPsqlExtensions) ++ - (makeOurOrioleDbPostgresPkgs version postgresql); - - pgbin = oriole_pkgs.buildEnv { - name = "postgresql-and-plugins-${postgresql.version}"; - paths = [ postgresql ] ++ selectedPackages; - buildInputs = [ oriole_pkgs.makeWrapper ]; - passthru = postgresql.passthru // { - inherit postgresql; - args = selectedPackages; - }; - - postBuild = '' - # Ensure all necessary directories exist - mkdir -p $out/bin - mkdir -p $out/lib/postgresql - mkdir -p $out/share/postgresql/extension - - # Create proper links for all binaries with environment - for bin in ${postgresql}/bin/*; do - binary_name=$(basename $bin) - rm -f "$out/bin/$binary_name" - makeWrapper $bin $out/bin/$binary_name \ - --set POSTGRES_MAJOR_VERSION "${postgresql.psqlSchema}" \ - --prefix PATH : $out/bin \ - --prefix LD_LIBRARY_PATH : "$out/lib:$out/lib/postgresql" \ - --prefix PKG_CONFIG_PATH : "$out/lib/pkgconfig" \ - --set PGDATA "/var/lib/postgresql/${postgresql.psqlSchema}" \ - --set PGHOST "/run/postgresql" - done - - # Ensure extension files are properly linked - for pkg in ${postgresql} ${toString selectedPackages}; do - # Link shared objects - if [ -d "$pkg/lib" ]; then - find "$pkg/lib" -type f -name '*.so' -exec ln -sf {} "$out/lib/postgresql/" \; - fi - - # Link extension control and SQL files - if [ -d "$pkg/share/postgresql/extension" ]; then - find "$pkg/share/postgresql/extension" -type f \( -name '*.control' -o -name '*.sql' \) \ - -exec ln -sf {} "$out/share/postgresql/extension/" \; - fi - done - - # Create pg_config if it doesn't exist - if [ ! -e "$out/bin/pg_config" ]; then - makeWrapper ${postgresql}/bin/pg_config $out/bin/pg_config \ - --set prefix "$out" \ - --set includedir "$out/include" \ - --set pkgincludedir "$out/include/postgresql" \ - --set includedir-server "$out/include/postgresql/server" \ - --set libdir "$out/lib" \ - --set pkglibdir "$out/lib/postgresql" \ - --set localedir "$out/share/locale" \ - --set mandir "$out/share/man" \ - --set sharedir "$out/share/postgresql" \ - --set sysconfdir "/etc/postgresql" \ - --set pgxs "$out/lib/postgresql/pgxs/src/makefiles/pgxs.mk" \ - --set configure "--enable-nls --with-openssl --with-libxml --with-libxslt" - fi - ''; - } // { - version = postgresql.version; - }; - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin upstreamExts ourExts) ]; - }; - # Create an attribute set, containing all the relevant packages for a # PostgreSQL install, wrapped up with a bow on top. There are three # packages: @@ -368,11 +227,6 @@ makeOrioleDbPostgresBin = version: patchedPostgres: exts = makeOurPostgresPkgsSet version; recurseForDerivations = true; }; - makeOrioleDbPostgres = version: patchedPostgres: rec { - bin = makeOrioleDbPostgresBin version patchedPostgres; - exts = makeOurOrioleDbPostgresPkgsSet version patchedPostgres; - recurseForDerivations = true; - }; # The base set of packages that we export from this Nix Flake, that can # be used with 'nix build'. Don't use the names listed below; check the @@ -390,7 +244,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_oriole-16 = makeOrioleDbPostgres "16" postgresql_orioledb_16; + psql_orioledb-16 = makePostgres "orioledb-16" ; }; # Find the active PostgreSQL version @@ -406,20 +260,20 @@ makeOrioleDbPostgresBin = version: patchedPostgres: }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; - postgresql_oriole-16 = postgresql_orioledb_16; + postgresql_orioledb-16 = getPostgresqlPackage "orioledb-16"; in postgresVersions //{ supabase-groonga = supabase-groonga; # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - psql_oriole-16 = postgresVersions.psql_oriole-16; + psql_orioledb-16 = postgresVersions.psql_orioledb-16; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16 postgresql_oriole-16; + inherit postgresql_15 postgresql_16 postgresql_orioledb-16; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; - postgresql_oriole-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb_16.debug else null; + postgresql_orioledb-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-16.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -464,9 +318,9 @@ makeOrioleDbPostgresBin = version: patchedPostgres: platforms = platforms.all; }; }; - postgresql_oriole-16_src = pkgs.stdenv.mkDerivation { + postgresql_orioledb-16_src = pkgs.stdenv.mkDerivation { pname = "postgresql-16-src"; - version = postgresql_oriole-16.version; + version = postgresql_orioledb-16.version; src = postgresql_16.src; @@ -545,7 +399,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ @@ -575,7 +429,7 @@ makeOrioleDbPostgresBin = version: patchedPostgres: --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_oriole-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 5f92262b2..484cac777 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -1,11 +1,10 @@ self: let - #adapted from the postgresql nixpkgs package versions = { postgresql_15 = ./15.nix; postgresql_16 = ./16.nix; + postgresql_orioledb-16 = ./orioledb-16.nix; }; - mkAttributes = jitSupport: self.lib.mapAttrs' (version: path: let @@ -15,7 +14,6 @@ let inherit jitSupport self; }) ) versions; - in # variations without and with JIT (mkAttributes false) // (mkAttributes true) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 323b65167..c03d1bcb3 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -7,6 +7,8 @@ let , glibc, zlib, readline, openssl, icu, lz4, zstd, systemd, libossp_uuid , pkg-config, libxml2, tzdata, libkrb5, substituteAll, darwin , linux-pam + #orioledb specific + , perl, bison, flex, docbook_xsl, docbook_xml_dtd_45, docbook_xsl_ns, libxslt # This is important to obtain a version of `libpq` that does not depend on systemd. , systemdSupport ? lib.meta.availableOn stdenv.hostPlatform systemd && !stdenv.hostPlatform.isStatic @@ -49,10 +51,16 @@ let inherit version; pname = pname + lib.optionalString jitSupport "-jit"; - src = fetchurl { - url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; - inherit hash; - }; + src = if (builtins.match "16_.*" version != null) then + fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; + inherit hash; + } + else + fetchurl { + url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; + inherit hash; + }; hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; @@ -74,7 +82,10 @@ let ++ lib.optionals pythonSupport [ python3 ] ++ lib.optionals gssSupport [ libkrb5 ] ++ lib.optionals stdenv'.isLinux [ linux-pam ] - ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ]; + ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] + ++ lib.optionals (builtins.match "16_.*" version != null) [ + perl bison flex docbook_xsl docbook_xml_dtd_45 docbook_xsl_ns libxslt + ]; nativeBuildInputs = [ makeWrapper diff --git a/nix/postgresql/orioledb-16.nix b/nix/postgresql/orioledb-16.nix new file mode 100644 index 000000000..50f8961f0 --- /dev/null +++ b/nix/postgresql/orioledb-16.nix @@ -0,0 +1,20 @@ +# { self, fetchurl, ... }: + +# let +# generic = import ./generic.nix rec { +# version = "16"; +# hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; +# src = fetchurl { +# url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; +# sha256 = hash; +# }; +# }; +# in +# generic.overrideAttrs (oldAttrs: { +# inherit generic; +# }) +# orioledb-16.nix +import ./generic.nix { + version = "16_31"; + hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; +} From 1cd20632576b47d5937561fa1f9c20a6e47ccf01 Mon Sep 17 00:00:00 2001 From: samrose Date: Wed, 23 Oct 2024 19:16:12 -0400 Subject: [PATCH 039/109] fix: update name for bundle --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 78c843cd1..7213333f0 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,7 +10,7 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_oriole-16/bin -o psql_orioledb_16 +nix build .#psql_orioledb-16/bin -o psql_orioledb_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 From 0e23e7d700ad673ae71458ca7dfe2620fc4d2258 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 23 Oct 2024 21:40:14 -0400 Subject: [PATCH 040/109] chore: small cleanup --- flake.nix | 6 +----- nix/postgresql/orioledb-16.nix | 16 ---------------- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/flake.nix b/flake.nix index 5d25c3ef7..eb590dc38 100644 --- a/flake.nix +++ b/flake.nix @@ -33,7 +33,7 @@ }; inherit system; overlays = [ - # NOTE (aseipp): add any needed overlays here. in theory we could + # NOTE: add any needed overlays here. in theory we could # pull them from the overlays/ directory automatically, but we don't # want to have an arbitrary order, since it might matter. being # explicit is better. @@ -162,10 +162,6 @@ }; }; - # makeOurOrioleDbPostgresPkgs = version: patchedPostgres: - # let postgresql = patchedPostgres; - # in map (path: pkgs.callPackage path { inherit postgresql; }) orioledbExtension; - makeOurPostgresPkgs = version: let postgresql = getPostgresqlPackage version; diff --git a/nix/postgresql/orioledb-16.nix b/nix/postgresql/orioledb-16.nix index 50f8961f0..e9a66850e 100644 --- a/nix/postgresql/orioledb-16.nix +++ b/nix/postgresql/orioledb-16.nix @@ -1,19 +1,3 @@ -# { self, fetchurl, ... }: - -# let -# generic = import ./generic.nix rec { -# version = "16"; -# hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; -# src = fetchurl { -# url = "https://github.com/orioledb/postgres/archive/refs/tags/patches16_31.tar.gz"; -# sha256 = hash; -# }; -# }; -# in -# generic.overrideAttrs (oldAttrs: { -# inherit generic; -# }) -# orioledb-16.nix import ./generic.nix { version = "16_31"; hash = "sha256-29uHUACwZKh8e4zJ9tWzEhLNjEuh6P31KbpxnMEhtuI="; From 9d48e503545709094b1eb9f336a898c5414fcd11 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 23 Oct 2024 23:27:26 -0400 Subject: [PATCH 041/109] chore: bump version release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 996049d78..b33eae2b0 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-7" + postgresoriole-16: "oriole-16.3.1.000-staging-8" # Non Postgres Extensions pgbouncer_release: "1.19.0" From fed90a0a1319057d56713c5c72b25a21d4e6058c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 00:30:52 -0400 Subject: [PATCH 042/109] chore: cleanup names --- ansible/tasks/stage2-setup-postgres.yml | 10 +++++----- ansible/tasks/test-image.yml | 4 ++-- ansible/vars.yml | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index dbc61e5fd..685aa651d 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -6,16 +6,16 @@ - name: Check psql_version and modify supautils.conf and postgresql.conf if necessary block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Remove specified extensions from postgresql.conf if oriole-16 build ansible.builtin.command: cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_oriole_16 and stage2_nix + when: is_psql_orioledb_16 and stage2_nix become: yes - name: Remove specified extensions from supautils.conf if oriole-16 build @@ -222,9 +222,9 @@ - name: Check psql_version and run postgis linking if not oriole-16 block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir shell: > diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index f342d2e00..40a208136 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -65,9 +65,9 @@ - name: Check psql_version and modify migrations if orioe-16 block: - - name: Check if psql_version is psql_oriole-16 + - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_oriole-16' }}" + is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" - name: Remove specified extensions from SQL file ansible.builtin.command: diff --git a/ansible/vars.yml b/ansible/vars.yml index b33eae2b0..41af00bde 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-8" + postgresoriole-16: "oriole-16.3.1.000-staging-9" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 9b078d03e1ecffeb53ead597d8be216e96ccf436 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 00:33:15 -0400 Subject: [PATCH 043/109] chore: correct names --- ansible/vars.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 41af00bde..e16af209a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -5,14 +5,14 @@ async_mode: true postgres_major: #- "15" #- "16" - - "oriole-16" + - "orioledb-16" # Full version strings for each major version # TOD PR uncomment these lines postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "oriole-16.3.1.000-staging-9" + postgresoriole-16: "orioledb-16.3.1.000-staging-9" # Non Postgres Extensions pgbouncer_release: "1.19.0" From bc2b4cc592cd8517391c61c4e5a9b31fe1257b1c Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 06:10:56 -0400 Subject: [PATCH 044/109] fix: var name --- ansible/tasks/stage2-setup-postgres.yml | 4 ++-- ansible/vars.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 685aa651d..f1082347f 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -15,7 +15,7 @@ cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_orioledb_16 and stage2_nix + when: is_psql_oriole_16 and stage2_nix become: yes - name: Remove specified extensions from supautils.conf if oriole-16 build @@ -280,4 +280,4 @@ ansible.builtin.lineinfile: path: /etc/environment.d/postgresql.env line: 'GRN_PLUGINS_DIR=/var/lib/postgresql/.nix-profile/lib/groonga/plugins' - become: yes \ No newline at end of file + become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index e16af209a..74e26989e 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "orioledb-16.3.1.000-staging-9" + postgresoriole-16: "orioledb-16.3.1.000-staging-10" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 12dc39708bd0eddb70398b9c92e22b38bd0115e7 Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 06:25:10 -0400 Subject: [PATCH 045/109] fix: one more var name --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 74e26989e..d427d64b1 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresoriole-16: "orioledb-16.3.1.000-staging-10" + postgresorioledb-16: "orioledb-16.3.1.000-staging-10" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 73944e19e18608ca0d666d65a0c5eb76b48efa02 Mon Sep 17 00:00:00 2001 From: samrose Date: Thu, 24 Oct 2024 07:31:31 -0400 Subject: [PATCH 046/109] chore: bump to release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d427d64b1..8a7a11b85 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresorioledb-16: "orioledb-16.3.1.000-staging-10" + postgresorioledb-16: "orioledb-16.3.1.000-staging-11" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 9ae9c9f9a08c8b77d85e7a3a213ddb344f20ee4c Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 12:17:46 -0400 Subject: [PATCH 047/109] fix: reformatting metadata for clikchouse dep as git was not able to fetch --- ansible/vars.yml | 2 +- nix/ext/wrappers/default.nix | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 8a7a11b85..14b35ef69 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_major: postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresorioledb-16: "orioledb-16.3.1.000-staging-11" + postgresorioledb-16: "orioledb-16.3.1.000-staging-12" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 74b94f9f9..d53604a5d 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -49,10 +49,46 @@ buildPgrxExtension_0_12_6 rec { CARGO_BUILD_JOBS = "2"; CARGO="${cargo}/bin/cargo"; + #CARGO_NET_GIT_FETCH_WITH_CLI = "true"; cargoLock = { lockFile = "${src}/Cargo.lock"; allowBuiltinFetchGit = true; }; + + preConfigure = '' + cd wrappers + + # update the clickhouse-rs dependency + # append the branch name to the git URL to help cargo locate the commit + # while maintaining the rev for reproducibility + awk -i inplace ' + /\[dependencies.clickhouse-rs\]/ { + print + getline + if ($0 ~ /git =/) { + print "git = \"https://github.com/suharev7/clickhouse-rs/async-await\"" + } else { + print + } + while ($0 !~ /^\[/ && NF > 0) { + getline + if ($0 ~ /rev =/) print + if ($0 ~ /^\[/) print + } + next + } + { print } + ' Cargo.toml + + # Verify the file is still valid TOML, break build with this error + # if it is not + if ! cargo verify-project 2>/dev/null; then + echo "Failed to maintain valid TOML syntax" + exit 1 + fi + + cd .. + ''; buildAndTestSubdir = "wrappers"; buildFeatures = [ From fa05170681d03b89a2bf33f537b50fc5cf6b8d35 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 24 Oct 2024 13:11:41 -0400 Subject: [PATCH 048/109] feat: wip refactor to pg 17 for orioledb --- .github/workflows/ami-release-nix.yml | 2 +- ansible/tasks/stage2-setup-postgres.yml | 10 ++++----- ansible/tasks/test-image.yml | 8 +++---- ansible/vars.yml | 5 +++-- docker/nix/build_nix.sh | 3 ++- flake.nix | 29 +++++++++++++------------ nix/ext/orioledb.nix | 6 ++--- nix/postgresql/default.nix | 1 + nix/postgresql/generic.nix | 2 +- nix/postgresql/orioledb-17.nix | 4 ++++ nix/tools/run-client.sh.in | 6 ++--- nix/tools/run-server.sh.in | 10 ++++----- 12 files changed, 47 insertions(+), 39 deletions(-) create mode 100644 nix/postgresql/orioledb-17.nix diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index f7005a4b0..5464c4507 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -5,7 +5,7 @@ on: branches: #- develop #- release/* - - sam/oriole16 + - sam/oriole17 paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index f1082347f..d605973b6 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -8,14 +8,14 @@ block: - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" - name: Remove specified extensions from postgresql.conf if oriole-16 build ansible.builtin.command: cmd: > sed -i 's/ timescaledb,//g' /etc/postgresql/postgresql.conf - when: is_psql_oriole_16 and stage2_nix + when: is_psql_oriole and stage2_nix become: yes - name: Remove specified extensions from supautils.conf if oriole-16 build @@ -23,7 +23,7 @@ cmd: > sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' /etc/postgresql-custom/supautils.conf - when: is_psql_oriole_16 and stage2_nix + when: is_psql_oriole and stage2_nix become: yes - name: Install Postgres from nix binary cache @@ -224,7 +224,7 @@ block: - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" + is_psql_oriole: "{{ psql_version == 'psql_orioledb-16' }}" - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir shell: > @@ -232,7 +232,7 @@ sudo find /var/lib/postgresql/.nix-profile/share/postgresql/contrib/ -mindepth 1 -type d -exec sh -c 'for dir do sudo ln -s "$dir" "/usr/lib/postgresql/share/postgresql/contrib/$(basename "$dir")"; done' sh {} + \ && chown -R postgres:postgres "/usr/lib/postgresql/share/postgresql/contrib/" become: yes - when: stage2_nix and not is_psql_oriole_16 + when: stage2_nix and not is_psql_oriole - name: Create symbolic links from /var/lib/postgresql/.nix-profile/share/postgresql/timezonesets to /usr/lib/postgresql/share/postgresql/timeszonesets file: diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 40a208136..ab9a10030 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -67,13 +67,13 @@ block: - name: Check if psql_version is psql_orioledb-16 set_fact: - is_psql_oriole_16: "{{ psql_version == 'psql_orioledb-16' }}" + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql - when: is_psql_oriole_16 + when: is_psql_oriole become: yes - name: Remove specified extension files from extensions directory @@ -86,14 +86,14 @@ - '*plv8*.sql' - '*pgvector*.sql' register: files_to_remove - when: is_psql_oriole_16 + when: is_psql_oriole - name: Delete matched extension files ansible.builtin.file: path: "{{ item.path }}" state: absent loop: "{{ files_to_remove.files }}" - when: is_psql_oriole_16 + when: is_psql_oriole become: yes diff --git a/ansible/vars.yml b/ansible/vars.yml index 14b35ef69..e217f6aa1 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -5,14 +5,15 @@ async_mode: true postgres_major: #- "15" #- "16" - - "orioledb-16" + - "orioledb-17" # Full version strings for each major version # TOD PR uncomment these lines postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" - postgresorioledb-16: "orioledb-16.3.1.000-staging-12" + #postgresorioledb-16: "orioledb-16.3.1.000-staging-12" + postgresorioledb-17: "orioledb-17.0.1.000-staging-1" # Non Postgres Extensions pgbouncer_release: "1.19.0" diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 7213333f0..d4423fda5 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,7 +10,8 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_orioledb-16/bin -o psql_orioledb_16 +#nix build .#psql_orioledb-16/bin -o psql_orioledb_16 +nix build .#psql_orioledb-17/bin -o psql_orioledb_17 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 diff --git a/flake.nix b/flake.nix index ef1324eb8..55d6fe544 100644 --- a/flake.nix +++ b/flake.nix @@ -160,9 +160,7 @@ orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; getPostgresqlPackage = version: - if version == "orioledb_16" - then pkgs.postgresql."postgresql_orioledb-16" - else pkgs.postgresql."postgresql_${version}"; + pkgs.postgresql."postgresql_${version}"; # Create a 'receipt' file for a given postgresql package. This is a way # of adding a bit of metadata to the package, which can be used by other # tools to inspect what the contents of the install are: the PSQL @@ -202,7 +200,7 @@ makeOurPostgresPkgs = version: let postgresql = getPostgresqlPackage version; - extensionsToUse = if version == "orioledb-16" + extensionsToUse = if (builtins.elem version ["orioledb-16" "orioledb-17"]) then orioledbExtensions else ourExtensions; in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; @@ -277,7 +275,8 @@ postgresVersions = { psql_15 = makePostgres "15"; psql_16 = makePostgres "16"; - psql_orioledb-16 = makePostgres "orioledb-16" ; + #psql_orioledb-16 = makePostgres "orioledb-16" ; + psql_orioledb-17 = makePostgres "orioledb-17" ; }; # Find the active PostgreSQL version @@ -293,7 +292,8 @@ }; postgresql_15 = getPostgresqlPackage "15"; postgresql_16 = getPostgresqlPackage "16"; - postgresql_orioledb-16 = getPostgresqlPackage "orioledb-16"; + #postgresql_orioledb-16 = getPostgresqlPackage "orioledb-16"; + postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; in postgresVersions //{ supabase-groonga = supabase-groonga; @@ -302,13 +302,14 @@ # PostgreSQL versions. psql_15 = postgresVersions.psql_15; psql_16 = postgresVersions.psql_16; - psql_orioledb-16 = postgresVersions.psql_orioledb-16; + #psql_orioledb-16 = postgresVersions.psql_orioledb-16; + psql_orioledb-17 = postgresVersions.psql_orioledb-17; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_16 postgresql_orioledb-16; + inherit postgresql_15 postgresql_16 postgresql_orioledb-17; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_16_debug = if pkgs.stdenv.isLinux then postgresql_16.debug else null; - postgresql_orioledb-16_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-16.debug else null; + postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = postgresql_15.version; @@ -353,11 +354,11 @@ platforms = platforms.all; }; }; - postgresql_orioledb-16_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-16-src"; - version = postgresql_orioledb-16.version; + postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { + pname = "postgresql-17-src"; + version = postgresql_orioledb-17.version; - src = postgresql_16.src; + src = postgresql_orioledb-17.src; nativeBuildInputs = [ pkgs.bzip2 ]; @@ -434,7 +435,7 @@ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL_CONF_FILE' $out/etc/postgresql/postgresql.conf \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ --subst-var-by 'READREPL_CONF_FILE' "$out/etc/postgresql-custom/read-replica.conf" \ --subst-var-by 'LOGGING_CONF_FILE' "$out/etc/postgresql-custom/logging.conf" \ diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index ff7c12a96..7df4301e2 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,10 +6,10 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "bd8e32d0ebaafd0ea3ec3074233b65167f3b6fb7"; - sha256 = "sha256-bzH1SgPZ6q90HpqRsECY2XQPghEcd2Hg4X55G43unNo="; + rev = "2861cf9091a92bcabbbbf1f4d15f8edff480fea1"; + sha256 = ""; }; - version = "patches16_31"; + version = "patches17_3"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=31"; installPhase = '' diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index 484cac777..a065d5aa7 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -4,6 +4,7 @@ let postgresql_15 = ./15.nix; postgresql_16 = ./16.nix; postgresql_orioledb-16 = ./orioledb-16.nix; + postgresql_orioledb-17 = ./orioledb-17.nix; }; mkAttributes = jitSupport: self.lib.mapAttrs' (version: path: diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index c03d1bcb3..22e55df4e 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -51,7 +51,7 @@ let inherit version; pname = pname + lib.optionalString jitSupport "-jit"; - src = if (builtins.match "16_.*" version != null) then + src = if (builtins.match "17_*" version != null) then fetchurl { url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; inherit hash; diff --git a/nix/postgresql/orioledb-17.nix b/nix/postgresql/orioledb-17.nix new file mode 100644 index 000000000..bcf436f55 --- /dev/null +++ b/nix/postgresql/orioledb-17.nix @@ -0,0 +1,4 @@ +import ./generic.nix { + version = "17_3"; + hash = ""; +} diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in index 309588828..3c4e9181e 100644 --- a/nix/tools/run-client.sh.in +++ b/nix/tools/run-client.sh.in @@ -92,9 +92,9 @@ elif [ "$PSQL_VERSION" == "16" ]; then echo "Starting client for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" -elif [ "$PSQL_VERSION" == "orioledb-16" ]; then - echo "Starting client for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ +elif [ "$PSQL_VERSION" == "orioledb-17" ]; then + echo "Starting client for PSQL ORIOLEDB 17" + PSQLORIOLEDB16=@PSQLORIOLEDB17_BINDIR@ BINDIR="$PSQLORIOLEDB16" else echo "Please provide a valid Postgres version (15, 16, or orioledb-16)" diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 085b480bc..a7e266d18 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -10,10 +10,10 @@ elif [ "$1" == "16" ]; then echo "Starting server for PSQL 16" PSQL16=@PSQL16_BINDIR@ BINDIR="$PSQL16" -elif [ "$1" == "orioledb-16" ]; then - echo "Starting server for PSQL ORIOLEDB 16" - PSQLORIOLEDB16=@PSQLORIOLEDB16_BINDIR@ - BINDIR="$PSQLORIOLEDB16" +elif [ "$1" == "orioledb-17" ]; then + echo "Starting server for PSQL ORIOLEDB 17" + PSQLORIOLEDB17=@PSQLORIOLEDB17_BINDIR@ + BINDIR="$PSQLORIOLEDB17" else echo "Please provide a valid Postgres version (15, 16 or orioledb-16)" exit 1 @@ -65,7 +65,7 @@ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" remove_config_items() { - if [ "$1" == "orioledb-16" ]; then + if [ "$1" == "orioledb-17" ]; then # Remove items from postgresql.conf sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" From 0e283e6b63b12add961b5aa75790b984f86b2a50 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 25 Oct 2024 15:06:26 -0400 Subject: [PATCH 049/109] feat: working orioledb-17 --- flake.nix | 8 ++++++-- nix/ext/orioledb.nix | 4 ++-- nix/ext/pgaudit.nix | 4 ++++ nix/postgresql/generic.nix | 12 ++++++------ nix/postgresql/orioledb-17.nix | 2 +- nix/tools/run-server.sh.in | 2 ++ 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/flake.nix b/flake.nix index 55d6fe544..9e26c692a 100644 --- a/flake.nix +++ b/flake.nix @@ -154,7 +154,11 @@ x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && - x != ./nix/ext/pgrouting.nix + x != ./nix/ext/pgrouting.nix && + x != ./nix/ext/pg_jsonschema.nix && + x != ./nix/ext/pg_graphql.nix && + x != ./nix/ext/rum.nix && + x != ./nix/ext/wal2json.nix ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; @@ -466,7 +470,7 @@ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL16_BINDIR' '${basePackages.psql_16.bin}' \ - --subst-var-by 'PSQLORIOLEDB16_BINDIR' '${basePackages.psql_orioledb-16.bin}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 7df4301e2..efe2de413 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -7,11 +7,11 @@ stdenv.mkDerivation rec { owner = "orioledb"; repo = "orioledb"; rev = "2861cf9091a92bcabbbbf1f4d15f8edff480fea1"; - sha256 = ""; + sha256 = "sha256-VVOHOIhQSQ824vfZlptOj1Gcu6Jg80amjRp7CpUOq6s="; }; version = "patches17_3"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=31"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=3"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index f6c5d8bb0..84fbf440d 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -2,6 +2,10 @@ #adapted from https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/sql/postgresql/ext/pgaudit.nix let source = { + "17" = { + version = "17.0"; + hash = "sha256-3ksq09wiudQPuBQI3dhEQi8IkXKLVIsPFgBnwLiicro="; + }; "16" = { version = "16.0"; hash = "sha256-8+tGOl1U5y9Zgu+9O5UDDE4bec4B0JC/BQ6GLhHzQzc="; diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index 22e55df4e..19b73c162 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -51,7 +51,7 @@ let inherit version; pname = pname + lib.optionalString jitSupport "-jit"; - src = if (builtins.match "17_*" version != null) then + src = if (builtins.match "[0-9][0-9]_.*" version != null) then fetchurl { url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; inherit hash; @@ -64,7 +64,7 @@ let hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; - outputs = [ "out" "lib" "doc" "man" ]; + outputs = [ "out" "lib" ]; setOutputFlags = false; # $out retains configureFlags :-/ buildInputs = [ @@ -83,7 +83,7 @@ let ++ lib.optionals gssSupport [ libkrb5 ] ++ lib.optionals stdenv'.isLinux [ linux-pam ] ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] - ++ lib.optionals (builtins.match "16_.*" version != null) [ + ++ lib.optionals (builtins.match "[0-9][0-9]_.*" version != null) [ perl bison flex docbook_xsl docbook_xml_dtd_45 docbook_xsl_ns libxslt ]; @@ -97,7 +97,7 @@ let separateDebugInfo = true; - buildFlags = [ "world" ]; + buildFlags = [ "world-bin" ]; # Makes cross-compiling work when xml2-config can't be executed on the host. # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 @@ -138,7 +138,7 @@ let (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch) ]; - installTargets = [ "install-world" ]; + installTargets = [ "install-world-bin" ]; postPatch = '' # Hardcode the path to pgxs so pg_config returns the path in $out @@ -288,7 +288,7 @@ let paths = f pkgs ++ [ postgresql postgresql.lib - postgresql.man # in case user installs this into environment + #TODO RM postgresql.man # in case user installs this into environment ]; nativeBuildInputs = [ makeWrapper ]; diff --git a/nix/postgresql/orioledb-17.nix b/nix/postgresql/orioledb-17.nix index bcf436f55..82ff44e65 100644 --- a/nix/postgresql/orioledb-17.nix +++ b/nix/postgresql/orioledb-17.nix @@ -1,4 +1,4 @@ import ./generic.nix { version = "17_3"; - hash = ""; + hash = "sha256-LOlST80eU+Wmmcmo4WHGwOClYZ+bBahyltsGYeKWCU4="; } diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index a7e266d18..c802a560e 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -68,6 +68,8 @@ remove_config_items() { if [ "$1" == "orioledb-17" ]; then # Remove items from postgresql.conf sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" fi } From 051c9fd9e2fd800aae1336042cc08537e42fda05 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sun, 27 Oct 2024 08:36:59 -0400 Subject: [PATCH 050/109] fix: increase role duration to avoid expiration --- .github/workflows/nix-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml index 08c316bce..79d3f1870 100644 --- a/.github/workflows/nix-build.yml +++ b/.github/workflows/nix-build.yml @@ -40,6 +40,7 @@ jobs: role-to-assume: ${{ secrets.DEV_AWS_ROLE }} aws-region: "us-east-1" output-credentials: true + role-duration-seconds: 7200 - name: write secret key # use python so we don't interpolate the secret into the workflow logs, in case of bugs run: | From 33b89482e5d5a171cc2ef4711afce79e015e85a2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Sun, 27 Oct 2024 12:44:04 -0400 Subject: [PATCH 051/109] fix: correct version --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index d4423fda5..d26bd60ee 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -14,4 +14,4 @@ nix build .#psql_16/bin -o psql_16 nix build .#psql_orioledb-17/bin -o psql_orioledb_17 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_16 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 From 230c7c766eb8786a2801184a2a76302d69b82dda Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 09:42:10 -0400 Subject: [PATCH 052/109] fix: clickhouse needs git in build inputs --- ansible/tasks/test-image.yml | 4 ++-- nix/ext/wrappers/default.nix | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index ab9a10030..4e4c37e43 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -63,9 +63,9 @@ failed_when: retval.rc != 0 when: debpkg_mode or stage2_nix -- name: Check psql_version and modify migrations if orioe-16 +- name: Check psql_version and modify migrations if oriole-xx block: - - name: Check if psql_version is psql_orioledb-16 + - name: Check if psql_version is psql_orioledb-xx set_fact: is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index d53604a5d..c36e1a522 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -9,6 +9,7 @@ , darwin , jq , rust-bin +, git }: let rustVersion = "1.80.0"; @@ -29,7 +30,7 @@ buildPgrxExtension_0_12_6 rec { }; nativeBuildInputs = [ pkg-config cargo ]; - buildInputs = [ openssl postgresql ] ++ lib.optionals (stdenv.isDarwin) [ + buildInputs = [ openssl postgresql git ] ++ lib.optionals (stdenv.isDarwin) [ darwin.apple_sdk.frameworks.CoreFoundation darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.SystemConfiguration From ee4ce323af7ec2a235bdaf6a569426f31c6eb768 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 10:37:56 -0400 Subject: [PATCH 053/109] fix: install and then remove git for clickhouse dep --- ansible/tasks/stage2-setup-postgres.yml | 18 ++++++++++++++++++ nix/ext/wrappers/default.nix | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d605973b6..922e8d471 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -26,6 +26,15 @@ when: is_psql_oriole and stage2_nix become: yes +- name: Install Git for Nix package management + become: yes + apt: + name: git + state: present + update_cache: yes + when: stage2_nix + + - name: Install Postgres from nix binary cache become: yes shell: | @@ -56,6 +65,15 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" when: stage2_nix +- name: Remove Git after Nix package installations + become: yes + apt: + name: git + state: absent + autoremove: yes + purge: yes + when: stage2_nix + - name: Set ownership and permissions for /etc/ssl/private become: yes file: diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index c36e1a522..3bcd294df 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -30,7 +30,7 @@ buildPgrxExtension_0_12_6 rec { }; nativeBuildInputs = [ pkg-config cargo ]; - buildInputs = [ openssl postgresql git ] ++ lib.optionals (stdenv.isDarwin) [ + buildInputs = [ openssl postgresql ] ++ lib.optionals (stdenv.isDarwin) [ darwin.apple_sdk.frameworks.CoreFoundation darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.SystemConfiguration From 2ccb5ac171aafe64e646e1b41d9a401a06925d8b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 11:53:09 -0400 Subject: [PATCH 054/109] fix: correct rules for version --- ansible/tasks/stage2-setup-postgres.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 922e8d471..f84d9543f 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -240,9 +240,9 @@ - name: Check psql_version and run postgis linking if not oriole-16 block: - - name: Check if psql_version is psql_orioledb-16 + - name: Check if psql_version is psql_orioledb-17 set_fact: - is_psql_oriole: "{{ psql_version == 'psql_orioledb-16' }}" + is_psql_oriole: "{{ psql_version == 'psql_orioledb-17' }}" - name: Recursively create symbolic links and set permissions for the contrib/postgis-* dir shell: > From f9cadcc4afcf83dd8f4aff841a213797fb2155f5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 14:42:54 -0400 Subject: [PATCH 055/109] fix: db_user_namespace was deprecated starting in pg 16.4 and higher --- ansible/tasks/stage2-setup-postgres.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index f84d9543f..2179661c8 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -25,7 +25,13 @@ /etc/postgresql-custom/supautils.conf when: is_psql_oriole and stage2_nix become: yes - + - name: Remove db_user_namespace from postgresql.conf if oriole-xx build + ansible.builtin.command: + cmd: > + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes - name: Install Git for Nix package management become: yes apt: From 1c9c1f13419de0d0ea2822fa61d03a31ebc684bc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 15:45:05 -0400 Subject: [PATCH 056/109] apply fix for wrappers build, deactivate ext in unit test --- ansible/tasks/test-image.yml | 19 +++++++++++++------ nix/ext/wrappers/default.nix | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 4e4c37e43..0cd8a5713 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -57,11 +57,6 @@ LOCALE_ARCHIVE: /usr/lib/locale/locale-archive when: stage2_nix -- name: Run Unit tests (with filename unit-test-*) on Postgres Database - shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql - register: retval - failed_when: retval.rc != 0 - when: debpkg_mode or stage2_nix - name: Check psql_version and modify migrations if oriole-xx block: @@ -95,7 +90,19 @@ loop: "{{ files_to_remove.files }}" when: is_psql_oriole become: yes - + + - name: Remove specified extensions from SQL file + ansible.builtin.command: + cmd: > + sed -i "/'pg_graphql',/d" /tmp/unit-tests/unit-test-01.sql + when: is_psql_oriole + become: yes + +- name: Run Unit tests (with filename unit-test-*) on Postgres Database + shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql + register: retval + failed_when: retval.rc != 0 + when: debpkg_mode or stage2_nix - name: Run migrations tests shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 3bcd294df..c36e1a522 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -30,7 +30,7 @@ buildPgrxExtension_0_12_6 rec { }; nativeBuildInputs = [ pkg-config cargo ]; - buildInputs = [ openssl postgresql ] ++ lib.optionals (stdenv.isDarwin) [ + buildInputs = [ openssl postgresql git ] ++ lib.optionals (stdenv.isDarwin) [ darwin.apple_sdk.frameworks.CoreFoundation darwin.apple_sdk.frameworks.Security darwin.apple_sdk.frameworks.SystemConfiguration From 2f0186f5c2f2930f49ea35dc9fb621328906fe9d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 17:02:20 -0400 Subject: [PATCH 057/109] chore: deactivate more tests due to deactivated extensions --- ansible/tasks/test-image.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 0cd8a5713..a2007e35b 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -80,6 +80,10 @@ - '*pgrouting*.sql' - '*plv8*.sql' - '*pgvector*.sql' + - '*wal2json*.sql' + - '*rum.sql' + - '*pg_jsonschema*.sql' + - '*pg_graphql*.sql' register: files_to_remove when: is_psql_oriole From 4ac77c40bf4dd95edef7bd183b6b67d7aef77ba1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 18:08:30 -0400 Subject: [PATCH 058/109] fix: also remove from sql file --- ansible/tasks/test-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index a2007e35b..605ca0c63 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -67,7 +67,7 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\wal2json\|rum\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql when: is_psql_oriole become: yes From 61434e08161d26cba58115d9ed73e623f06176ae Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 18:53:49 -0400 Subject: [PATCH 059/109] fix: regex --- ansible/tasks/test-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 605ca0c63..e55ae0bd6 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -67,7 +67,7 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\wal2json\|rum\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\|wal2json\|rum\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql when: is_psql_oriole become: yes From 24b7442c77d96409eb5447c24fc39f4223966c58 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 20:39:40 -0400 Subject: [PATCH 060/109] fix: also remove ref here --- ansible/tasks/test-image.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index e55ae0bd6..68ff3d599 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -101,7 +101,15 @@ sed -i "/'pg_graphql',/d" /tmp/unit-tests/unit-test-01.sql when: is_psql_oriole become: yes - + + - name: Remove graphql schema test + lineinfile: + path: /tmp/migrations/tests/databases/exists.sql + regexp: "^SELECT has_schema\\('graphql'\\);$" + state: absent + become: yes + when: is_psql_oriole + - name: Run Unit tests (with filename unit-test-*) on Postgres Database shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql register: retval From 9ec0bfa170df0c92f2299b4999353517d44974b9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 21:35:04 -0400 Subject: [PATCH 061/109] fix: graphql_public schema too --- ansible/tasks/test-image.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 68ff3d599..5afdbed14 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -110,6 +110,14 @@ become: yes when: is_psql_oriole + - name: Remove graphql schema test + lineinfile: + path: /tmp/migrations/tests/databases/exists.sql + regexp: "^SELECT has_schema\\('graphql_public'\\);$" + state: absent + become: yes + when: is_psql_oriole + - name: Run Unit tests (with filename unit-test-*) on Postgres Database shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql register: retval From 37d6e494ef4e9a3cf1f28e71f38ce76b8f60d20e Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 21:39:33 -0400 Subject: [PATCH 062/109] chore: correct dir --- ansible/tasks/test-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index 5afdbed14..a531d7599 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -104,7 +104,7 @@ - name: Remove graphql schema test lineinfile: - path: /tmp/migrations/tests/databases/exists.sql + path: /tmp/migrations/tests/database/exists.sql regexp: "^SELECT has_schema\\('graphql'\\);$" state: absent become: yes @@ -112,7 +112,7 @@ - name: Remove graphql schema test lineinfile: - path: /tmp/migrations/tests/databases/exists.sql + path: /tmp/migrations/tests/database/exists.sql regexp: "^SELECT has_schema\\('graphql_public'\\);$" state: absent become: yes From 45de660610ccdb8a95fd0e0f6e650eb322a9e966 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 28 Oct 2024 22:43:35 -0400 Subject: [PATCH 063/109] chore: staging release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e217f6aa1..d4895db47 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -13,7 +13,7 @@ postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" #postgresorioledb-16: "orioledb-16.3.1.000-staging-12" - postgresorioledb-17: "orioledb-17.0.1.000-staging-1" + postgresorioledb-17: "orioledb-17.0.1.000-staging-2" # Non Postgres Extensions pgbouncer_release: "1.19.0" From dd649cc6c25277dda5cb67126b24ffd5cf4766e4 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 29 Oct 2024 09:10:27 -0400 Subject: [PATCH 064/109] feat: re-introduce wal2json rum and pgvector --- ansible/tasks/test-image.yml | 5 +---- flake.nix | 8 ++++---- nix/ext/rum.nix | 4 ++-- nix/ext/wal2json.nix | 4 ++-- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/ansible/tasks/test-image.yml b/ansible/tasks/test-image.yml index a531d7599..d8e951a3e 100644 --- a/ansible/tasks/test-image.yml +++ b/ansible/tasks/test-image.yml @@ -67,7 +67,7 @@ - name: Remove specified extensions from SQL file ansible.builtin.command: cmd: > - sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pgvector\|wal2json\|rum\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql + sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\|pg_jsonschema\|pg_graphql\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql when: is_psql_oriole become: yes @@ -79,9 +79,6 @@ - '*postgis*.sql' - '*pgrouting*.sql' - '*plv8*.sql' - - '*pgvector*.sql' - - '*wal2json*.sql' - - '*rum.sql' - '*pg_jsonschema*.sql' - '*pg_graphql*.sql' register: files_to_remove diff --git a/flake.nix b/flake.nix index 9e26c692a..199b071e5 100644 --- a/flake.nix +++ b/flake.nix @@ -151,14 +151,14 @@ orioleFilteredExtensions = builtins.filter ( x: x != ./nix/ext/timescaledb.nix && - x != ./nix/ext/pgvector.nix && + #x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && x != ./nix/ext/pgrouting.nix && x != ./nix/ext/pg_jsonschema.nix && - x != ./nix/ext/pg_graphql.nix && - x != ./nix/ext/rum.nix && - x != ./nix/ext/wal2json.nix + x != ./nix/ext/pg_graphql.nix + #x != ./nix/ext/rum.nix + #x != ./nix/ext/wal2json.nix ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; diff --git a/nix/ext/rum.nix b/nix/ext/rum.nix index 16bf106c5..1270f1ff1 100644 --- a/nix/ext/rum.nix +++ b/nix/ext/rum.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "rum"; - version = "1.3.13"; + version = "1.3.14"; src = fetchFromGitHub { owner = "postgrespro"; repo = "rum"; rev = version; - hash = "sha256-yy2xeDnk3fENN+En0st4mv60nZlqPafIzwf68jwJ5fE="; + hash = "sha256-VsfpxQqRBu9bIAP+TfMRXd+B3hSjuhU2NsutocNiCt8="; }; buildInputs = [ postgresql ]; diff --git a/nix/ext/wal2json.nix b/nix/ext/wal2json.nix index 751eb64c0..ed578c7fd 100644 --- a/nix/ext/wal2json.nix +++ b/nix/ext/wal2json.nix @@ -2,13 +2,13 @@ stdenv.mkDerivation rec { pname = "wal2json"; - version = "2_5"; + version = "2_6"; src = fetchFromGitHub { owner = "eulerto"; repo = "wal2json"; rev = "wal2json_${builtins.replaceStrings ["."] ["_"] version}"; - hash = "sha256-Gpc9uDKrs/dmVSFgdgHM453+TaEnhRh9t0gDbSn8FUI="; + hash = "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM="; }; buildInputs = [ postgresql ]; From a1efc9b6cb52489117855ae6ab96c34d8be7fbc4 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 29 Oct 2024 09:12:49 -0400 Subject: [PATCH 065/109] fix: also restore pgvector in start-server --- flake.nix | 3 --- nix/tools/run-server.sh.in | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/flake.nix b/flake.nix index 199b071e5..2d8031f1a 100644 --- a/flake.nix +++ b/flake.nix @@ -151,14 +151,11 @@ orioleFilteredExtensions = builtins.filter ( x: x != ./nix/ext/timescaledb.nix && - #x != ./nix/ext/pgvector.nix && x != ./nix/ext/plv8.nix && x != ./nix/ext/postgis.nix && x != ./nix/ext/pgrouting.nix && x != ./nix/ext/pg_jsonschema.nix && x != ./nix/ext/pg_graphql.nix - #x != ./nix/ext/rum.nix - #x != ./nix/ext/wal2json.nix ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index c802a560e..7738beaea 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -70,7 +70,7 @@ remove_config_items() { sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" - sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" fi } remove_config_items "$1" From feec1e55572d1ad8d7a761cf28b276a3076aa467 Mon Sep 17 00:00:00 2001 From: samrose Date: Tue, 29 Oct 2024 10:13:52 -0400 Subject: [PATCH 066/109] chore: bump staging release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index d4895db47..e8a61a22d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -13,7 +13,7 @@ postgres_release: #postgres15: "15.8.1.003-staging-5" #postgres16: "16.3.1.000-staging-5" #postgresorioledb-16: "orioledb-16.3.1.000-staging-12" - postgresorioledb-17: "orioledb-17.0.1.000-staging-2" + postgresorioledb-17: "orioledb-17.0.1.000-staging-3" # Non Postgres Extensions pgbouncer_release: "1.19.0" From 14448821edaf206fb40ec9a50b475675ea66ed16 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 29 Oct 2024 20:49:43 -0400 Subject: [PATCH 067/109] feat: oriole specific default settings --- ansible/tasks/stage2-setup-postgres.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 377d9c23f..cf716706c 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -32,6 +32,20 @@ /etc/postgresql/postgresql.conf when: is_psql_oriole and stage2_nix become: yes + - name: Append orioledb to shared_preload_libraries append within closing quote + ansible.builtin.command: + cmd: > + sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + - name: Add default_table_access_method setting + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "default_table_access_method = 'orioledb'" + state: present + when: is_psql_oriole and stage2_nix + become: yes - name: Install Git for Nix package management become: yes From 2fb7958aa900952ff1c12b7976beaa59a649cd29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 30 Oct 2024 11:55:08 -0400 Subject: [PATCH 068/109] feat: use icu for locales only if building orioledb --- ansible/tasks/setup-postgres.yml | 54 +++++++++++++++++-------- ansible/tasks/stage2-setup-postgres.yml | 15 +++++++ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 0038e3afc..116262eef 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -204,23 +204,43 @@ ansible_command_timeout: 60 when: debpkg_mode -- name: Initialize the database stage2_nix - become: yes - become_user: postgres - shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" - args: - executable: /bin/bash - environment: - LANG: en_US.UTF-8 - LANGUAGE: en_US.UTF-8 - LC_ALL: en_US.UTF-8 - LC_CTYPE: en_US.UTF-8 - LOCALE_ARCHIVE: /usr/lib/locale/locale-archive - vars: - ansible_command_timeout: 60 - # Circumvents the following error: - # "Timeout (12s) waiting for privilege escalation prompt" - when: stage2_nix +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary + block: + - name: Check if psql_version is psql_orioledb + set_fact: + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" + + - name: Initialize the database stage2_nix (non-orioledb) + become: yes + become_user: postgres + shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + vars: + ansible_command_timeout: 60 + when: stage2_nix and not is_psql_oriole + + - name: Initialize the database stage2_nix (orioledb) + become: yes + become_user: postgres + shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" -o "--locale-provider=icu" -o "--encoding=UTF-8" -o "--icu-locale=en_US.UTF-8" + args: + executable: /bin/bash + environment: + LANG: en_US.UTF-8 + LANGUAGE: en_US.UTF-8 + LC_ALL: en_US.UTF-8 + LC_CTYPE: en_US.UTF-8 + LOCALE_ARCHIVE: /usr/lib/locale/locale-archive + vars: + ansible_command_timeout: 60 + when: stage2_nix and is_psql_oriole - name: copy PG systemd unit template: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index cf716706c..ca7b73219 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -25,6 +25,7 @@ /etc/postgresql-custom/supautils.conf when: is_psql_oriole and stage2_nix become: yes + - name: Remove db_user_namespace from postgresql.conf if oriole-xx build ansible.builtin.command: cmd: > @@ -32,6 +33,7 @@ /etc/postgresql/postgresql.conf when: is_psql_oriole and stage2_nix become: yes + - name: Append orioledb to shared_preload_libraries append within closing quote ansible.builtin.command: cmd: > @@ -39,6 +41,7 @@ /etc/postgresql/postgresql.conf when: is_psql_oriole and stage2_nix become: yes + - name: Add default_table_access_method setting ansible.builtin.lineinfile: path: /etc/postgresql/postgresql.conf @@ -47,6 +50,18 @@ when: is_psql_oriole and stage2_nix become: yes + - name: Add ICU locale settings + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "{{ item }}" + state: present + with_items: + - "lc_collate = 'en-US-x-icu'" + - "lc_ctype = 'en-US-x-icu'" + - "icu_locale = 'en-US'" + when: is_psql_oriole and stage2_nix + become: yes + - name: Install Git for Nix package management become: yes apt: From 2ff3c015d13a8568ae72af6e11fe57917c91e484 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 30 Oct 2024 13:47:52 -0400 Subject: [PATCH 069/109] fix: need var in stage 1 too --- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 45cbc8ac6..5991df87c 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -214,7 +214,10 @@ EOF # Run Ansible playbook #export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_DEBUG=True && export ANSIBLE_REMOTE_TEMP=/mnt/tmp export ANSIBLE_LOG_PATH=/tmp/ansible.log && export ANSIBLE_REMOTE_TEMP=/mnt/tmp - ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false}' $ARGS + ansible-playbook -c chroot -i '/mnt,' /tmp/ansible-playbook/ansible/playbook.yml \ + --extra-vars '{"nixpkg_mode": true, "debpkg_mode": false, "stage2_nix": false} ' \ + --extra-vars "psql_version=psql_${POSTGRES_MAJOR_VERSION}" \ + $ARGS } function update_systemd_services { From 18d91169c098a70552d5f3329a9edba489346cd8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 30 Oct 2024 14:41:37 -0400 Subject: [PATCH 070/109] fix: settings must be in db init --- ansible/tasks/setup-postgres.yml | 12 +++++++++--- ansible/tasks/stage2-setup-postgres.yml | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 116262eef..ed27c31ff 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -229,9 +229,15 @@ - name: Initialize the database stage2_nix (orioledb) become: yes become_user: postgres - shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" -o "--locale-provider=icu" -o "--encoding=UTF-8" -o "--icu-locale=en_US.UTF-8" - args: - executable: /bin/bash + cmd: > + source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb \ + -o "--allow-group-access" \ + -o "--username=supabase_admin" \ + -o "--locale-provider=icu" \ + -o "--encoding=UTF-8" \ + -o "--icu-locale=en_US.UTF-8" \ + -o "--lc-collate=en-US-x-icu" \ + -o "--lc-ctype=en-US-x-icu" environment: LANG: en_US.UTF-8 LANGUAGE: en_US.UTF-8 diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index ca7b73219..17b61267a 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -50,17 +50,18 @@ when: is_psql_oriole and stage2_nix become: yes - - name: Add ICU locale settings - ansible.builtin.lineinfile: - path: /etc/postgresql/postgresql.conf - line: "{{ item }}" - state: present - with_items: - - "lc_collate = 'en-US-x-icu'" - - "lc_ctype = 'en-US-x-icu'" - - "icu_locale = 'en-US'" - when: is_psql_oriole and stage2_nix - become: yes + # TODO PR may need to remove this. + # - name: Add ICU locale settings + # ansible.builtin.lineinfile: + # path: /etc/postgresql/postgresql.conf + # line: "{{ item }}" + # state: present + # with_items: + # - "lc_collate = 'en-US-x-icu'" + # - "lc_ctype = 'en-US-x-icu'" + # - "icu_locale = 'en-US'" + # when: is_psql_oriole and stage2_nix + # become: yes - name: Install Git for Nix package management become: yes From 1d6c728c21b2b14907c5a19f8d2fca8ecc7d36a0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 30 Oct 2024 16:02:15 -0400 Subject: [PATCH 071/109] fix: ansible args --- ansible/tasks/setup-postgres.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index ed27c31ff..06a6b549c 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -213,7 +213,7 @@ - name: Initialize the database stage2_nix (non-orioledb) become: yes become_user: postgres - shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" + shell: source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data -o "--allow-group-access" -o "--username=supabase_admin" args: executable: /bin/bash environment: @@ -229,15 +229,17 @@ - name: Initialize the database stage2_nix (orioledb) become: yes become_user: postgres - cmd: > - source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb \ - -o "--allow-group-access" \ - -o "--username=supabase_admin" \ - -o "--locale-provider=icu" \ - -o "--encoding=UTF-8" \ - -o "--icu-locale=en_US.UTF-8" \ - -o "--lc-collate=en-US-x-icu" \ - -o "--lc-ctype=en-US-x-icu" + shell: > + source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data + --allow-group-access + --username=supabase_admin + --locale-provider=icu + --encoding=UTF-8 + --icu-locale=en_US.UTF-8 + --lc-collate=en-US-x-icu + --lc-ctype=en-US-x-icu + args: + executable: /bin/bash environment: LANG: en_US.UTF-8 LANGUAGE: en_US.UTF-8 From 376f8d9fc9257281132b8c45b7055c11ad2dd881 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 30 Oct 2024 16:58:55 -0400 Subject: [PATCH 072/109] fix: PostgreSQL uses ICU for all locale-related operations, so we don't need to specify the collation and character type settings separately. --- ansible/tasks/setup-postgres.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 06a6b549c..69b78c2a7 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -236,8 +236,6 @@ --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8 - --lc-collate=en-US-x-icu - --lc-ctype=en-US-x-icu args: executable: /bin/bash environment: From c2312c3ff9f42a5cee7b2df0201a5c96126a6a81 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 31 Oct 2024 01:56:00 +0000 Subject: [PATCH 073/109] chore: add a migration for orioledb activation by default --- .../db/migrations/20241031003909_create_orioledb.sql | 10 ++++++++++ nix/tools/run-server.sh.in | 7 +++++-- 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 migrations/db/migrations/20241031003909_create_orioledb.sql diff --git a/migrations/db/migrations/20241031003909_create_orioledb.sql b/migrations/db/migrations/20241031003909_create_orioledb.sql new file mode 100644 index 000000000..cf4b0e642 --- /dev/null +++ b/migrations/db/migrations/20241031003909_create_orioledb.sql @@ -0,0 +1,10 @@ +-- migrate:up +do $$ +begin + if not exists (select 1 from pg_extension where extname = 'orioledb') then + create extension if not exists orioledb; + end if; +end $$; + +-- migrate:down +drop extension if exists orioledb; diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 7738beaea..c9c23849a 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -64,16 +64,19 @@ pgsodium.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" -remove_config_items() { +orioledb_config_items() { if [ "$1" == "orioledb-17" ]; then # Remove items from postgresql.conf sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATADIR/postgresql.conf" + echo "default_table_access_method = 'orioledb'" >> "$DATADIR/postgresql.conf" + fi } -remove_config_items "$1" +orioledb_config_items "$1" if [ "$CURRENT_SYSTEM" = "aarch64-darwin" ]; then echo "NOTE: using aarch64-darwin system" sed -i '' 's/ pg_net,//g' "$DATDIR/postgresql.conf" From cbcb962d57186027c06fd40ce2c5c3275702b483 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 31 Oct 2024 03:32:02 -0400 Subject: [PATCH 074/109] feat: turn on oriole if oriole ext exists --- ebssurrogate/files/unit-tests/unit-test-01.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ebssurrogate/files/unit-tests/unit-test-01.sql b/ebssurrogate/files/unit-tests/unit-test-01.sql index 0feb70e8b..11e2e9ce0 100644 --- a/ebssurrogate/files/unit-tests/unit-test-01.sql +++ b/ebssurrogate/files/unit-tests/unit-test-01.sql @@ -1,5 +1,12 @@ BEGIN; CREATE EXTENSION IF NOT EXISTS pgtap; +do $$ +begin + if not exists (select 1 from pg_extension where extname = 'orioledb') then + create extension if not exists orioledb; + end if; +end $$; + SELECT plan(8); -- Check installed extensions From c11d2e9bd1ee43dcca53341f1fcca03f61dd1c1f Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 31 Oct 2024 14:11:11 -0400 Subject: [PATCH 075/109] test: conditional on test for orioledb --- .../files/unit-tests/unit-test-01.sql | 68 +++++++++++++------ 1 file changed, 46 insertions(+), 22 deletions(-) diff --git a/ebssurrogate/files/unit-tests/unit-test-01.sql b/ebssurrogate/files/unit-tests/unit-test-01.sql index 11e2e9ce0..f3d47459f 100644 --- a/ebssurrogate/files/unit-tests/unit-test-01.sql +++ b/ebssurrogate/files/unit-tests/unit-test-01.sql @@ -1,40 +1,64 @@ BEGIN; CREATE EXTENSION IF NOT EXISTS pgtap; -do $$ -begin - if not exists (select 1 from pg_extension where extname = 'orioledb') then - create extension if not exists orioledb; - end if; -end $$; + +DO $$ +DECLARE + extension_array text[]; + orioledb_available boolean; +BEGIN + -- Check if orioledb is available + SELECT EXISTS ( + SELECT 1 FROM pg_available_extensions WHERE name = 'orioledb' + ) INTO orioledb_available; + + -- If available, create it and add to the expected extensions list + IF orioledb_available THEN + CREATE EXTENSION IF NOT EXISTS orioledb; + extension_array := ARRAY[ + 'plpgsql', + 'pg_stat_statements', + 'pgsodium', + 'pgtap', + 'pg_graphql', + 'pgcrypto', + 'pgjwt', + 'uuid-ossp', + 'supabase_vault', + 'orioledb' + ]; + ELSE + extension_array := ARRAY[ + 'plpgsql', + 'pg_stat_statements', + 'pgsodium', + 'pgtap', + 'pg_graphql', + 'pgcrypto', + 'pgjwt', + 'uuid-ossp', + 'supabase_vault' + ]; + END IF; + + -- Set the array as a temporary variable to use in the test + PERFORM set_config('myapp.extensions', array_to_string(extension_array, ','), false); +END $$; SELECT plan(8); --- Check installed extensions SELECT extensions_are( - ARRAY[ - 'plpgsql', - 'pg_stat_statements', - 'pgsodium', - 'pgtap', - 'pg_graphql', - 'pgcrypto', - 'pgjwt', - 'uuid-ossp', - 'supabase_vault' - ] + string_to_array(current_setting('myapp.extensions'), ',')::text[] ); --- Check schemas exists SELECT has_schema('pg_toast'); SELECT has_schema('pg_catalog'); SELECT has_schema('information_schema'); SELECT has_schema('public'); --- Check that service_role can execute certain pgsodium functions SELECT function_privs_are('pgsodium', 'crypto_aead_det_decrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); SELECT function_privs_are('pgsodium', 'crypto_aead_det_encrypt', array['bytea', 'bytea', 'uuid', 'bytea'], 'service_role', array['EXECUTE']); SELECT function_privs_are('pgsodium', 'crypto_aead_det_keygen', array[]::text[], 'service_role', array['EXECUTE']); -SELECT * from finish(); -ROLLBACK; +SELECT * FROM finish(); +ROLLBACK; \ No newline at end of file From 5bc0e33cc82309c3c4072d9d6b9f0d619b3712b1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 31 Oct 2024 16:15:02 -0400 Subject: [PATCH 076/109] fix: salt changed repo and method for adding apt package --- ansible/tasks/internal/install-salt.yml | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 281e370bb..4d75e4f56 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,14 +1,22 @@ - name: Add apt repository for Saltstack (arm) block: - - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg + - name: Ensure keyrings directory exists + ansible.builtin.file: + path: /etc/apt/keyrings + state: directory + mode: '0755' - - name: salt apt repo - ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg arch=arm64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/{{ salt_minion_version }} focal main - state: present + - name: Download Salt Project public key + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring.pgp + mode: '0644' + + - name: Add Salt repository sources + ansible.builtin.get_url: + url: https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources + dest: /etc/apt/sources.list.d/salt.sources + mode: '0644' when: platform == "arm64" - name: Add apt repository for Saltstack (amd) From a822ca4b4fb6e0a0e15b28334af09934454fbaf7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 31 Oct 2024 19:37:01 -0400 Subject: [PATCH 077/109] fix: udpate salt minion setup --- ansible/tasks/internal/install-salt.yml | 36 ++++++++++++++++--------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 4d75e4f56..e1e3a8a7a 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,37 +1,47 @@ - name: Add apt repository for Saltstack (arm) block: - - name: Ensure keyrings directory exists - ansible.builtin.file: + - name: Ensure /etc/apt/keyrings directory exists + file: path: /etc/apt/keyrings state: directory mode: '0755' - - name: Download Salt Project public key + - name: salt gpg key ansible.builtin.get_url: url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public - dest: /etc/apt/keyrings/salt-archive-keyring.pgp + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp mode: '0644' - - name: Add Salt repository sources - ansible.builtin.get_url: - url: https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources - dest: /etc/apt/sources.list.d/salt.sources - mode: '0644' + - name: salt apt repo + ansible.builtin.apt_repository: + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=arm64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' + state: present when: platform == "arm64" - name: Add apt repository for Saltstack (amd) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/{{ salt_minon_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "amd64" - name: Salt minion install apt: name: salt-minion + state: present + update_cache: yes \ No newline at end of file From 2b3df4b2e1fdf712650b7ee2d70f711b6101bb2d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 14:48:54 +0000 Subject: [PATCH 078/109] fix: silent skip if oriole not in the install --- .../db/migrations/20241031003909_create_orioledb.sql | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/migrations/db/migrations/20241031003909_create_orioledb.sql b/migrations/db/migrations/20241031003909_create_orioledb.sql index cf4b0e642..29bd0ae5f 100644 --- a/migrations/db/migrations/20241031003909_create_orioledb.sql +++ b/migrations/db/migrations/20241031003909_create_orioledb.sql @@ -1,10 +1,11 @@ --- migrate:up do $$ begin - if not exists (select 1 from pg_extension where extname = 'orioledb') then - create extension if not exists orioledb; + if exists (select 1 from pg_available_extensions where name = 'orioledb') then + if not exists (select 1 from pg_extension where extname = 'orioledb') then + create extension if not exists orioledb; + end if; end if; end $$; -- migrate:down -drop extension if exists orioledb; +drop extension if exists orioledb; \ No newline at end of file From 1f955df8eedda00e3d963f68163d513b43c35d49 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 09:52:18 -0500 Subject: [PATCH 079/109] test: check if oriole is available and if so then enable --- migrations/tests/test.sql | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/migrations/tests/test.sql b/migrations/tests/test.sql index 7afa40645..e19a6620d 100644 --- a/migrations/tests/test.sql +++ b/migrations/tests/test.sql @@ -5,6 +5,16 @@ BEGIN; CREATE EXTENSION IF NOT EXISTS pgtap; +-- Check and create OrioleDB if available +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_available_extensions WHERE name = 'orioledb') THEN + IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname = 'orioledb') THEN + CREATE EXTENSION orioledb; + END IF; + END IF; +END $$; + SELECT no_plan(); \ir fixtures.sql From 8ff1419c4c3613ec2a4504b2a84b90a6bcaa10ba Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 10:52:37 -0500 Subject: [PATCH 080/109] chore: build and cache src and debug pkgs as well --- docker/nix/build_nix.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index d26bd60ee..188030d65 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -12,6 +12,18 @@ nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 #nix build .#psql_orioledb-16/bin -o psql_orioledb_16 nix build .#psql_orioledb-17/bin -o psql_orioledb_17 +nix build .#postgresql_15_src -o psql_15_src +nix build .#postgresql_16_src -o psql_16_src +nix build .#postgresql_orioledb-17_src -o psql_orioledb_17_src +nix build .#postgresql_15_debug -o psql_15_debug +nix build .#postgresql_16_debug -o psql_16_debug +nix build .#postgresql_orioledb-17_debug -o psql_orioledb_17_debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_src +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_src +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_src +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_debug +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_debug +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_debug From c9da3ecc03fa816e2390f2a8c8e1c9c38725db66 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 11:40:46 -0500 Subject: [PATCH 081/109] fix: tmp supress build of src and debug --- docker/nix/build_nix.sh | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 188030d65..b5a87f1aa 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -10,20 +10,19 @@ nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '" #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -#nix build .#psql_orioledb-16/bin -o psql_orioledb_16 nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -nix build .#postgresql_15_src -o psql_15_src -nix build .#postgresql_16_src -o psql_16_src -nix build .#postgresql_orioledb-17_src -o psql_orioledb_17_src -nix build .#postgresql_15_debug -o psql_15_debug -nix build .#postgresql_16_debug -o psql_16_debug -nix build .#postgresql_orioledb-17_debug -o psql_orioledb_17_debug +# nix build .#postgresql_15_src -o psql_15_src +# nix build .#postgresql_16_src -o psql_16_src +# nix build .#postgresql_orioledb-17_src -o psql_orioledb_17_src +#nix build .#postgresql_15_debug -o psql_15_debug +#nix build .#postgresql_16_debug -o psql_16_debug +#nix build .#postgresql_orioledb-17_debug -o psql_orioledb_17_debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_src -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_src -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_src -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_debug -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_debug -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_debug +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_src +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_src +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_src +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_debug +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_debug +# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_debug From 78491e2246cb82f135577c5b210f5895c209930b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 11:54:31 -0500 Subject: [PATCH 082/109] chore: tmp disable src and debug --- ansible/tasks/stage2-setup-postgres.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 17b61267a..6e76b2521 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -89,17 +89,17 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supabase-groonga" when: stage2_nix -- name: Install debug symbols for postgres version - become: yes - shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" - when: stage2_nix +# - name: Install debug symbols for postgres version +# become: yes +# shell: | +# sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" +# when: stage2_nix -- name: Install source files for postgresql version - become: yes - shell: | - sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" - when: stage2_nix +# - name: Install source files for postgresql version +# become: yes +# shell: | +# sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" +# when: stage2_nix - name: Remove Git after Nix package installations become: yes From 41f291ea308cd4e7804467336389d6111bf06ca4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 12:54:09 -0500 Subject: [PATCH 083/109] fix: activate oriole first in the oriole context --- migrations/tests/test.sql | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/migrations/tests/test.sql b/migrations/tests/test.sql index e19a6620d..9682b4a2b 100644 --- a/migrations/tests/test.sql +++ b/migrations/tests/test.sql @@ -1,10 +1,3 @@ --- Create all extensions -\ir extensions/test.sql - -BEGIN; - -CREATE EXTENSION IF NOT EXISTS pgtap; - -- Check and create OrioleDB if available DO $$ BEGIN @@ -15,6 +8,13 @@ BEGIN END IF; END $$; +-- Create all extensions +\ir extensions/test.sql + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pgtap; + SELECT no_plan(); \ir fixtures.sql From c1962cba5fe48d934315345ed5095a046e28a5e0 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 14:15:12 -0500 Subject: [PATCH 084/109] fix: pgroonga build on macos --- flake.nix | 34 ++++++++++++++++++++-------------- nix/ext/pgroonga.nix | 24 +++++++++++++++++++++--- 2 files changed, 41 insertions(+), 17 deletions(-) diff --git a/flake.nix b/flake.nix index 65790a89c..293f5f316 100644 --- a/flake.nix +++ b/flake.nix @@ -528,40 +528,46 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; pg_regress = basePackages.pg_regress; + tmpDirCmd = if pkgs.stdenv.isDarwin then + "$(pwd)/postgres-tmp" + else + "mktemp -d"; in pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove pg_regress procps supabase-groonga ]; } '' - TMPDIR=$(mktemp -d) + $WORKDIR=$(${tmpDirCmd}) if [ $? -ne 0 ]; then echo "Failed to create temp directory" >&2 exit 1 fi + chmod 700 "$WORKDIR" + # Ensure the temporary directory is removed on exit - trap 'rm -rf "$TMPDIR"' EXIT + #trap 'rm -rf "$WORKDIR"' EXIT - export PGDATA="$TMPDIR/pgdata" - export PGSODIUM_DIR="$TMPDIR/pgsodium" + export PGDATA="$WORKDIR/pgdata" + export PGSODIUM_DIR="$WORKDIR/pgsodium" mkdir -p $PGDATA - mkdir -p $TMPDIR/logfile + mkdir -p $WORKDIR/logfile # Generate a random key and store it in an environment variable export PGSODIUM_KEY=$(head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n') export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins # Create a simple script to echo the key - echo '#!/bin/sh' > $TMPDIR/getkey.sh - echo 'echo $PGSODIUM_KEY' >> $TMPDIR/getkey.sh - chmod +x $TMPDIR/getkey.sh + echo '#!/bin/sh' > $WORKDIR/getkey.sh + echo 'echo $PGSODIUM_KEY' >> $WORKDIR/getkey.sh + chmod +x $WORKDIR/getkey.sh initdb --locale=C --username=supabase_admin substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "$TMPDIR/getkey.sh" + --subst-var-by PGSODIUM_GETKEY_SCRIPT "$WORKDIR/getkey.sh" echo "listen_addresses = '*'" >> $PGDATA/postgresql.conf echo "port = 5432" >> $PGDATA/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf - #postgres -D "$PGDATA" -k "$TMPDIR" -h localhost -p 5432 >$TMPDIR/logfile/postgresql.log 2>&1 & - pg_ctl -D "$PGDATA" -l $TMPDIR/logfile/postgresql.log -o "-k $TMPDIR -p 5432" start + #postgres -D "$PGDATA" -k "$WORKDIR" -h localhost -p 5432 >$WORKDIR/logfile/postgresql.log 2>&1 & + pg_ctl -D "$PGDATA" -l $WORKDIR/logfile/postgresql.log -o "-k $WORKDIR -p 5432" start for i in {1..60}; do if pg_isready -h localhost -p 5432; then echo "PostgreSQL is ready" @@ -573,14 +579,14 @@ echo "PostgreSQL status:" pg_ctl -D "$PGDATA" status echo "PostgreSQL log content:" - cat $TMPDIR/logfile/postgresql.log + cat $WORKDIR/logfile/postgresql.log exit 1 fi done createdb -p 5432 -h localhost --username=supabase_admin testing if ! psql -p 5432 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then echo "Error executing SQL file. PostgreSQL log content:" - cat $TMPDIR/logfile/postgresql.log + cat $WORKDIR/logfile/postgresql.log pg_ctl -D "$PGDATA" stop exit 1 fi @@ -598,7 +604,7 @@ $(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort ) pg_ctl -D "$PGDATA" stop - mv $TMPDIR/logfile/postgresql.log $out + mv $WORKDIR/logfile/postgresql.log $out echo ${pgpkg} ''; in diff --git a/nix/ext/pgroonga.nix b/nix/ext/pgroonga.nix index 884b54582..c7fe90bb1 100644 --- a/nix/ext/pgroonga.nix +++ b/nix/ext/pgroonga.nix @@ -1,4 +1,4 @@ -{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper }: +{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper, xxHash }: let supabase-groonga = callPackage ../supabase-groonga.nix { }; in @@ -10,7 +10,11 @@ stdenv.mkDerivation rec { sha256 = "sha256-iF/zh4zDDpAw5fxW1WG8i2bfPt4VYsnYArwOoE/lwgM="; }; nativeBuildInputs = [ pkg-config makeWrapper ]; - buildInputs = [ postgresql msgpack-c supabase-groonga mecab ]; + + buildInputs = [ postgresql msgpack-c supabase-groonga mecab ] ++ lib.optionals stdenv.isDarwin [ + xxHash + ]; + propagatedBuildInputs = [ supabase-groonga ]; configureFlags = [ "--with-mecab=${mecab}" @@ -19,16 +23,30 @@ stdenv.mkDerivation rec { "--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins" ]; - makeFlags = [ + makeFlags = [ "HAVE_MSGPACK=1" "MSGPACK_PACKAGE_NAME=msgpack-c" "HAVE_MECAB=1" ]; + NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin (builtins.concatStringsSep " " [ + "-Wno-error=incompatible-function-pointer-types" + "-Wno-error=format" + "-Wno-format" + "-I${supabase-groonga}/include/groonga" + "-I${xxHash}/include" + "-DPGRN_VERSION=\"${version}\"" + ]); + preConfigure = '' export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga" export GROONGA_CFLAGS="-I${supabase-groonga}/include" export MECAB_CONFIG="${mecab}/bin/mecab-config" + ${lib.optionalString stdenv.isDarwin '' + export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\"" + export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format" + ''} ''; installPhase = '' From d58f7e69f76621bc3e6bb308a5d13c86ae9ec8e2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 15:43:37 -0500 Subject: [PATCH 085/109] fix: restore flake check for now --- flake.nix | 36 +++++++++++++++--------------------- 1 file changed, 15 insertions(+), 21 deletions(-) diff --git a/flake.nix b/flake.nix index 293f5f316..a11db6595 100644 --- a/flake.nix +++ b/flake.nix @@ -528,46 +528,40 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; pg_regress = basePackages.pg_regress; - tmpDirCmd = if pkgs.stdenv.isDarwin then - "$(pwd)/postgres-tmp" - else - "mktemp -d"; in pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove pg_regress procps supabase-groonga ]; } '' - $WORKDIR=$(${tmpDirCmd}) + TMPDIR=$(mktemp -d) if [ $? -ne 0 ]; then echo "Failed to create temp directory" >&2 exit 1 fi - chmod 700 "$WORKDIR" - # Ensure the temporary directory is removed on exit - #trap 'rm -rf "$WORKDIR"' EXIT + trap 'rm -rf "$TMPDIR"' EXIT - export PGDATA="$WORKDIR/pgdata" - export PGSODIUM_DIR="$WORKDIR/pgsodium" + export PGDATA="$TMPDIR/pgdata" + export PGSODIUM_DIR="$TMPDIR/pgsodium" mkdir -p $PGDATA - mkdir -p $WORKDIR/logfile + mkdir -p $TMPDIR/logfile # Generate a random key and store it in an environment variable export PGSODIUM_KEY=$(head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n') export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins # Create a simple script to echo the key - echo '#!/bin/sh' > $WORKDIR/getkey.sh - echo 'echo $PGSODIUM_KEY' >> $WORKDIR/getkey.sh - chmod +x $WORKDIR/getkey.sh + echo '#!/bin/sh' > $TMPDIR/getkey.sh + echo 'echo $PGSODIUM_KEY' >> $TMPDIR/getkey.sh + chmod +x $TMPDIR/getkey.sh initdb --locale=C --username=supabase_admin substitute ${./nix/tests/postgresql.conf.in} $PGDATA/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "$WORKDIR/getkey.sh" + --subst-var-by PGSODIUM_GETKEY_SCRIPT "$TMPDIR/getkey.sh" echo "listen_addresses = '*'" >> $PGDATA/postgresql.conf echo "port = 5432" >> $PGDATA/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf - #postgres -D "$PGDATA" -k "$WORKDIR" -h localhost -p 5432 >$WORKDIR/logfile/postgresql.log 2>&1 & - pg_ctl -D "$PGDATA" -l $WORKDIR/logfile/postgresql.log -o "-k $WORKDIR -p 5432" start + #postgres -D "$PGDATA" -k "$TMPDIR" -h localhost -p 5432 >$TMPDIR/logfile/postgresql.log 2>&1 & + pg_ctl -D "$PGDATA" -l $TMPDIR/logfile/postgresql.log -o "-k $TMPDIR -p 5432" start for i in {1..60}; do if pg_isready -h localhost -p 5432; then echo "PostgreSQL is ready" @@ -579,14 +573,14 @@ echo "PostgreSQL status:" pg_ctl -D "$PGDATA" status echo "PostgreSQL log content:" - cat $WORKDIR/logfile/postgresql.log + cat $TMPDIR/logfile/postgresql.log exit 1 fi done createdb -p 5432 -h localhost --username=supabase_admin testing if ! psql -p 5432 -h localhost --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xaf ${./nix/tests/prime.sql}; then echo "Error executing SQL file. PostgreSQL log content:" - cat $WORKDIR/logfile/postgresql.log + cat $TMPDIR/logfile/postgresql.log pg_ctl -D "$PGDATA" stop exit 1 fi @@ -604,9 +598,9 @@ $(ls ${./nix/tests/sql} | sed -e 's/\..*$//' | sort ) pg_ctl -D "$PGDATA" stop - mv $WORKDIR/logfile/postgresql.log $out + mv $TMPDIR/logfile/postgresql.log $out echo ${pgpkg} - ''; + ''; in rec { # The list of all packages that can be built with 'nix build'. The list From 9abe2c9a13439e9444c94d3f06d46619fc163c74 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 18:25:59 -0500 Subject: [PATCH 086/109] fixing flake check for darwin + passing check until pg_net --- docker/nix/build_nix.sh | 7 +++++-- flake.nix | 20 ++++++++++++++++++-- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index b5a87f1aa..b216314f9 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,8 +5,11 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link +if [ "$(uname -sm)" != "Darwin arm64" ]; then + echo "Running on non-macOS arm64 and amd64, remove me when pg_net supports darwin arm64" + nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link + nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link +fi #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 diff --git a/flake.nix b/flake.nix index a11db6595..d10aa9ebf 100644 --- a/flake.nix +++ b/flake.nix @@ -528,19 +528,24 @@ pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; pg_regress = basePackages.pg_regress; + tmpDirCmd = if pkgs.stdenv.isDarwin then + ''mkdir -p /tmp/postgres-check.$$ && echo "/tmp/postgres-check.$$"'' + else + "mktemp -d"; in pkgs.runCommand "postgres-${pgpkg.version}-check-harness" { nativeBuildInputs = with pkgs; [ coreutils bash pgpkg pg_prove pg_regress procps supabase-groonga ]; } '' - TMPDIR=$(mktemp -d) + TMPDIR=$(${tmpDirCmd}) if [ $? -ne 0 ]; then echo "Failed to create temp directory" >&2 exit 1 fi + chmod -R 755 "$TMPDIR" # Ensure the temporary directory is removed on exit - trap 'rm -rf "$TMPDIR"' EXIT + #trap 'rm -rf "$TMPDIR"' EXIT export PGDATA="$TMPDIR/pgdata" export PGSODIUM_DIR="$TMPDIR/pgsodium" @@ -560,8 +565,19 @@ echo "listen_addresses = '*'" >> $PGDATA/postgresql.conf echo "port = 5432" >> $PGDATA/postgresql.conf echo "host all all 127.0.0.1/32 trust" >> $PGDATA/pg_hba.conf + # Add system-specific configuration for aarch64-darwin + #postgres -D "$PGDATA" -k "$TMPDIR" -h localhost -p 5432 >$TMPDIR/logfile/postgresql.log 2>&1 & pg_ctl -D "$PGDATA" -l $TMPDIR/logfile/postgresql.log -o "-k $TMPDIR -p 5432" start + # If server fails to start, output diagnostic information + if ! pg_ctl -D $TMPDIR/pgdata status > /dev/null 2>&1; then + echo "=== PostgreSQL Log ===" + cat $TMPDIR/logfile/postgresql.log + echo "=== System Information ===" + ls -la $TMPDIR/pgdata + whoami + groups + fi for i in {1..60}; do if pg_isready -h localhost -p 5432; then echo "PostgreSQL is ready" From e510e12563c84f51ac0c378f2ba2310524539dc7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 18:54:50 -0500 Subject: [PATCH 087/109] fix: start-server macos --- nix/tools/run-server.sh.in | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index c9c23849a..e48897150 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -65,7 +65,7 @@ session_preload_libraries = 'supautils'" \ "$PSQL_CONF_FILE" > "$DATDIR/postgresql.conf" orioledb_config_items() { - if [ "$1" == "orioledb-17" ]; then + if [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" != "aarch64-darwin" ]]; then # Remove items from postgresql.conf sed -i 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present @@ -75,6 +75,16 @@ orioledb_config_items() { echo "default_table_access_method = 'orioledb'" >> "$DATADIR/postgresql.conf" fi + if [[ "$1" = "orioledb-17" && "$CURRENT_SYSTEM" = "aarch64-darwin" ]]; then + # Remove items from postgresql.conf + sed -i '' 's/ timescaledb,//g;' "$DATDIR/postgresql.conf" + #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present + sed -i '' 's/db_user_namespace = off/#db_user_namespace = off/g;' "$DATDIR/postgresql.conf" + sed -i '' 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "$DATDIR/supautils.conf" + sed -i '' 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "$DATADIR/postgresql.conf" + echo "default_table_access_method = 'orioledb'" >> "$DATADIR/postgresql.conf" + + fi } orioledb_config_items "$1" if [ "$CURRENT_SYSTEM" = "aarch64-darwin" ]; then From 8cfa69c02ee4d87dc071d7babc115ab552eeedcd Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:16:09 -0500 Subject: [PATCH 088/109] fix: src yq from apt --- .github/workflows/testinfra-nix.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 4a51c159c..fbf0d7225 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -61,10 +61,15 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV + + - name: Install yq + run: | + sudo apt-get update + sudo apt-get install -y yq - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file From bdf31227b4dc36bfde30c68b16c1b4b8d13723ce Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:21:38 -0500 Subject: [PATCH 089/109] fix: all instances yq --- .github/workflows/testinfra-nix.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index fbf0d7225..3da799fd4 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -18,7 +18,7 @@ jobs: - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + VERSIONS=$(yq '.postgres_major[]' ansible/vars.yml | jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT test-ami-nix: @@ -61,11 +61,11 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - - name: Install yq + + - name: Install yq & jq run: | sudo apt-get update - sudo apt-get install -y yq + sudo apt-get install -y yq jq - name: Generate common-nix.vars.pkr.hcl run: | From a1b483f45f79ab1c857130ac803c556c39f24b12 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:28:23 -0500 Subject: [PATCH 090/109] fix: jq invoke --- .github/workflows/testinfra-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 3da799fd4..1c65553d9 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -18,7 +18,7 @@ jobs: - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(yq '.postgres_major[]' ansible/vars.yml | jq -- -R -s -c 'split("\n")[:-1]') + VERSIONS=$(yq '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT test-ami-nix: From 53b5f3ff11a7661080b5ac602e683093ebde55a6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:31:41 -0500 Subject: [PATCH 091/109] fix: nix profile install --- .github/workflows/testinfra-nix.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 1c65553d9..cf5672038 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -64,8 +64,7 @@ jobs: - name: Install yq & jq run: | - sudo apt-get update - sudo apt-get install -y yq jq + nix profile install nixpkgs#yq nixpkgs#jq - name: Generate common-nix.vars.pkr.hcl run: | From ec022a107d80aa7f53bb2acc365924055a6dcf04 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:38:48 -0500 Subject: [PATCH 092/109] fix: nix run --- .github/workflows/testinfra-nix.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index cf5672038..77f110296 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -18,7 +18,7 @@ jobs: - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(yq '.postgres_major[]' ansible/vars.yml | jq -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT test-ami-nix: @@ -62,13 +62,10 @@ jobs: - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Install yq & jq - run: | - nix profile install nixpkgs#yq nixpkgs#jq - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(yq '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file From 18661bf0a5dfdf43945d01aef0937f9130674d90 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 21:42:51 -0500 Subject: [PATCH 093/109] fix: sudo nix run --- .github/workflows/testinfra-nix.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/testinfra-nix.yml b/.github/workflows/testinfra-nix.yml index 77f110296..14db4a7f2 100644 --- a/.github/workflows/testinfra-nix.yml +++ b/.github/workflows/testinfra-nix.yml @@ -65,7 +65,7 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl # Ensure there's a newline at the end of the file From 39827b244026ea13664b37e804b89c1e16ab4bea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 4 Nov 2024 23:15:20 -0500 Subject: [PATCH 094/109] chore: cut staging release --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e17d94f08..b02da4f08 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,7 +10,7 @@ postgres_major: # Full version strings for each major version # TODO PR uncomment these lines postgres_release: - postgresorioledb-17: "orioledb-17.0.1.000-staging-3" + postgresorioledb-17: "orioledb-17.0.1.000-staging-4" #postgres15: "15.8.1.005" #postgres16: "16.3.1.011" From 7dd46e9ab0880c4b7aa98eac84f356ca7a9bc834 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 5 Nov 2024 17:22:31 -0500 Subject: [PATCH 095/109] chore: rm debugging --- flake.nix | 9 --------- 1 file changed, 9 deletions(-) diff --git a/flake.nix b/flake.nix index d10aa9ebf..f890cb650 100644 --- a/flake.nix +++ b/flake.nix @@ -569,15 +569,6 @@ #postgres -D "$PGDATA" -k "$TMPDIR" -h localhost -p 5432 >$TMPDIR/logfile/postgresql.log 2>&1 & pg_ctl -D "$PGDATA" -l $TMPDIR/logfile/postgresql.log -o "-k $TMPDIR -p 5432" start - # If server fails to start, output diagnostic information - if ! pg_ctl -D $TMPDIR/pgdata status > /dev/null 2>&1; then - echo "=== PostgreSQL Log ===" - cat $TMPDIR/logfile/postgresql.log - echo "=== System Information ===" - ls -la $TMPDIR/pgdata - whoami - groups - fi for i in {1..60}; do if pg_isready -h localhost -p 5432; then echo "PostgreSQL is ready" From 4775b44ba6cb59395a6bd3d043bf50bb856378f6 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 6 Nov 2024 09:51:55 -0500 Subject: [PATCH 096/109] fix: still need CURRENT_SYSTEM --- flake.nix | 3 ++- nix/tools/run-server.sh.in | 6 +----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/flake.nix b/flake.nix index 6b131ea08..a8b4eee53 100644 --- a/flake.nix +++ b/flake.nix @@ -439,7 +439,8 @@ --subst-var-by 'LOCALES' '${localeArchive}' \ --subst-var-by 'EXTENSION_CUSTOM_SCRIPTS_DIR' "$out/extension-custom-scripts" \ --subst-var-by 'MECAB_LIB' '${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so' \ - --subst-var-by 'GROONGA_DIR' '${supabase-groonga}' + --subst-var-by 'GROONGA_DIR' '${supabase-groonga}' \ + --subst-var-by 'CURRENT_SYSTEM' '${system}' chmod +x $out/bin/start-postgres-server ''; diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index 0d241eab9..942ef56b7 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -32,6 +32,7 @@ EXTENSION_CUSTOM_SCRIPTS=@EXTENSION_CUSTOM_SCRIPTS_DIR@ GROONGA=@GROONGA_DIR@ DATDIR=$(mktemp -d) LOCALE_ARCHIVE=@LOCALES@ +CURRENT_SYSTEM=@CURRENT_SYSTEM@ export LOCALE_ARCHIVE export LANG=en_US.UTF-8 export LANGUAGE=en_US.UTF-8 @@ -85,10 +86,5 @@ orioledb_config_items() { fi } orioledb_config_items "$1" -if [ "$CURRENT_SYSTEM" = "aarch64-darwin" ]; then - echo "NOTE: using aarch64-darwin system" - sed -i '' 's/ pg_net,//g' "$DATDIR/postgresql.conf" - sed -i '' 's/ pg_net,//g' "$DATDIR/supautils.conf" -fi export GRN_PLUGINS_DIR=$GROONGA/lib/groonga/plugins postgres --config-file="$DATDIR/postgresql.conf" -p "$PORTNO" -D "$DATDIR" -k /tmp From f075b020cd9ea3575a052f324bf88f7548f2be38 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 6 Nov 2024 11:45:09 -0500 Subject: [PATCH 097/109] test: re-introduce flake checks for 15 and 16 --- docker/nix/build_nix.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index b216314f9..502aef90c 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,11 +5,9 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi -if [ "$(uname -sm)" != "Darwin arm64" ]; then - echo "Running on non-macOS arm64 and amd64, remove me when pg_net supports darwin arm64" - nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link - nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link -fi +echo "Running on non-macOS arm64 and amd64, remove me when pg_net supports darwin arm64" +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link +nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link #no nix flake check on oriole yet nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 From d29bd9cec351dbffe9f25e2b5e3489009441e117 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Nov 2024 09:46:57 -0500 Subject: [PATCH 098/109] chore: bump versions of patched pg for oriole to 17_4 and ext to beta6 --- ansible/vars.yml | 2 +- nix/ext/orioledb.nix | 8 ++++---- nix/postgresql/orioledb-17.nix | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index b02da4f08..843bbecf6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,7 +10,7 @@ postgres_major: # Full version strings for each major version # TODO PR uncomment these lines postgres_release: - postgresorioledb-17: "orioledb-17.0.1.000-staging-4" + postgresorioledb-17: "orioledb-17.0.1.000-staging-5" #postgres15: "15.8.1.005" #postgres16: "16.3.1.011" diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index efe2de413..9775226e1 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -6,12 +6,12 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "orioledb"; repo = "orioledb"; - rev = "2861cf9091a92bcabbbbf1f4d15f8edff480fea1"; - sha256 = "sha256-VVOHOIhQSQ824vfZlptOj1Gcu6Jg80amjRp7CpUOq6s="; + rev = "beta6"; + sha256 = "sha256-7nmgQDVcNuaGOM6V9xrer2jqfxhJhXZBXRZoiPEoisE="; }; - version = "patches17_3"; + version = "beta6"; buildInputs = [ curl libkrb5 postgresql python3 openssl ]; - buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=3"; + buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=4"; installPhase = '' runHook preInstall mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/postgresql/orioledb-17.nix b/nix/postgresql/orioledb-17.nix index 82ff44e65..bbe5024a5 100644 --- a/nix/postgresql/orioledb-17.nix +++ b/nix/postgresql/orioledb-17.nix @@ -1,4 +1,4 @@ import ./generic.nix { - version = "17_3"; - hash = "sha256-LOlST80eU+Wmmcmo4WHGwOClYZ+bBahyltsGYeKWCU4="; + version = "17_4"; + hash = "sha256-qMJpTcw/ExipQ1XIPvYTQK+qgcDC7zecdLJDIuQug+0="; } From a5ca7b2228c7db420617b044388b33f123727634 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 15 Nov 2024 15:25:51 -0500 Subject: [PATCH 099/109] chore: bump var to release ami --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 10572e1f0..30d48db9a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -10,7 +10,7 @@ postgres_major: # Full version strings for each major version # TODO PR uncomment these lines postgres_release: - postgresorioledb-17: "orioledb-17.0.1.000-staging-5" + postgresorioledb-17: "orioledb-17.0.1.000-staging-6" #postgres15: "15.8.1.005" #postgres16: "16.3.1.011" From 32160915ff4774197314cd410bda75f2d045ab98 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 21 Nov 2024 10:22:44 -0500 Subject: [PATCH 100/109] chore: break down into functions --- ansible/files/postgres_prestart.sh.j2 | 44 +++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/ansible/files/postgres_prestart.sh.j2 b/ansible/files/postgres_prestart.sh.j2 index ae5aa1a7d..3ffe54c85 100644 --- a/ansible/files/postgres_prestart.sh.j2 +++ b/ansible/files/postgres_prestart.sh.j2 @@ -1,9 +1,49 @@ #!/bin/bash +check_orioledb_enabled() { + local pg_conf="/etc/postgresql/postgresql.conf" + if [ ! -f "$pg_conf" ]; then + return 0 + fi + grep "^shared_preload_libraries" "$pg_conf" | grep -c "orioledb" || return 0 +} + +get_shared_buffers() { + local opt_conf="/etc/postgresql-custom/generated-optimizations.conf" + if [ ! -f "$opt_conf" ]; then + return 0 + fi + grep "^shared_buffers = " "$opt_conf" | cut -d "=" -f2 | tr -d ' ' || return 0 +} + +update_orioledb_buffers() { + local pg_conf="/etc/postgresql/postgresql.conf" + local value="$1" + if grep -q "^orioledb.main_buffers = " "$pg_conf"; then + sed -i "s/^orioledb.main_buffers = .*/orioledb.main_buffers = $value/" "$pg_conf" + else + echo "orioledb.main_buffers = $value" >> "$pg_conf" + fi +} + +main() { + local has_orioledb=$(check_orioledb_enabled) + if [ "$has_orioledb" -lt 1 ]; then + return 0 + fi + local shared_buffers_value=$(get_shared_buffers) + if [ ! -z "$shared_buffers_value" ]; then + update_orioledb_buffers "$shared_buffers_value" + fi +} + +# Initial locale setup if [ $(cat /etc/locale.gen | grep -c en_US.UTF-8) -eq 0 ]; then - echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen + echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen fi if [ $(locale -a | grep -c en_US.utf8) -eq 0 ]; then - locale-gen + locale-gen fi + +main From 84d78d6c2fbbb9f9ef5ece15c1f6f58540994788 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Nov 2024 08:52:34 -0500 Subject: [PATCH 101/109] fix: skip x86 darwin for oriole nix builds --- docker/nix/build_nix.sh | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 502aef90c..0b254fd4e 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,25 +5,23 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi -echo "Running on non-macOS arm64 and amd64, remove me when pg_net supports darwin arm64" -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_15 -L --no-link -nix build .#checks.$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"').psql_16 -L --no-link -#no nix flake check on oriole yet + +SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') + +nix build .#checks.$SYSTEM.psql_15 -L --no-link +nix build .#checks.$SYSTEM.psql_16 -L --no-link + nix build .#psql_15/bin -o psql_15 nix build .#psql_16/bin -o psql_16 -nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -# nix build .#postgresql_15_src -o psql_15_src -# nix build .#postgresql_16_src -o psql_16_src -# nix build .#postgresql_orioledb-17_src -o psql_orioledb_17_src -#nix build .#postgresql_15_debug -o psql_15_debug -#nix build .#postgresql_16_debug -o psql_16_debug -#nix build .#postgresql_orioledb-17_debug -o psql_orioledb_17_debug + +# Skip orioledb-17 on x86_64-darwin +if [ "$SYSTEM" != "x86_64-darwin" ]; then + nix build .#psql_orioledb-17/bin -o psql_orioledb_17 +fi + +# Copy to S3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 -nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_src -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_src -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_src -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15_debug -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16_debug -# nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17_debug +if [ "$SYSTEM" != "x86_64-darwin" ]; then + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 +fi \ No newline at end of file From a0f32b491a9acff92591d53ea29ff236e2d85045 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Nov 2024 11:54:58 -0500 Subject: [PATCH 102/109] chore: cleanup repo for final review, rebase and merge --- .github/workflows/ami-release-nix.yml | 5 ++-- .github/workflows/test.yml | 38 +++++++++++++++++++++---- ansible/tasks/stage2-setup-postgres.yml | 14 --------- ansible/vars.yml | 11 ++++--- 4 files changed, 39 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index b09c6ae9a..3242a4e1d 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -3,9 +3,8 @@ name: Release AMI Nix on: push: branches: - #- develop - #- release/* - - sam/oriole17 + - develop + - release/* paths: - '.github/workflows/ami-release-nix.yml' - 'common-nix.vars.pkr.hcl' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c58df7d15..ab643cb49 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,8 +4,7 @@ on: push: branches: - develop - #TODO PR uncomment this line - #pull_request: + pull_request: workflow_dispatch: jobs: @@ -41,31 +40,45 @@ jobs: POSTGRES_PORT: 5478 POSTGRES_PASSWORD: password steps: + - name: Check if orioledb-17 + id: check-version + run: | + if [ "${{ matrix.postgres_version }}" = "\"orioledb-17\"" ]; then + echo "skip=true" >> $GITHUB_OUTPUT + else + echo "skip=false" >> $GITHUB_OUTPUT + fi + - uses: actions/checkout@v3 + if: steps.check-version.outputs.skip != 'true' - uses: DeterminateSystems/nix-installer-action@main + if: steps.check-version.outputs.skip != 'true' - name: Set PostgreSQL version environment variable + if: steps.check-version.outputs.skip != 'true' run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - name: Strip quotes from pg major and set env var + if: steps.check-version.outputs.skip != 'true' run: | stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl + if: steps.check-version.outputs.skip != 'true' run: | PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) - PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes + PG_VERSION=$(echo $PG_VERSION | tr -d '"') echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - id: settings - # Remove spaces and quotes to get the raw version string + if: steps.check-version.outputs.skip != 'true' run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - name: Generate args + if: steps.check-version.outputs.skip != 'true' id: args run: | ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) @@ -74,10 +87,15 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - run: docker context create builders + if: steps.check-version.outputs.skip != 'true' + - uses: docker/setup-buildx-action@v3 + if: steps.check-version.outputs.skip != 'true' with: endpoint: builders + - uses: docker/build-push-action@v5 + if: steps.check-version.outputs.skip != 'true' with: load: true context: . @@ -92,6 +110,7 @@ jobs: cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }} - name: Start Postgres + if: steps.check-version.outputs.skip != 'true' run: | docker run --rm --pull=never \ -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \ @@ -100,6 +119,7 @@ jobs: -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} - name: Install psql + if: steps.check-version.outputs.skip != 'true' run: | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - @@ -107,11 +127,13 @@ jobs: sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove + if: steps.check-version.outputs.skip != 'true' run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP env: SHELL: /bin/bash - name: Wait for healthy database + if: steps.check-version.outputs.skip != 'true' run: | count=0 until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do @@ -130,6 +152,7 @@ jobs: container: supabase_postgres - name: Run tests + if: steps.check-version.outputs.skip != 'true' run: pg_prove migrations/tests/test.sql env: PGHOST: localhost @@ -139,6 +162,7 @@ jobs: PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - name: Check migrations are idempotent + if: steps.check-version.outputs.skip != 'true' run: | for sql in ./migrations/db/migrations/*.sql; do echo "$0: running $sql" @@ -152,14 +176,16 @@ jobs: PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - name: Update Dockerfile.dbmate version + if: steps.check-version.outputs.skip != 'true' run: | sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - name: verify schema.sql is committed + if: steps.check-version.outputs.skip != 'true' run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql exit 1 - fi + fi \ No newline at end of file diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index 6e76b2521..f358c956a 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -50,19 +50,6 @@ when: is_psql_oriole and stage2_nix become: yes - # TODO PR may need to remove this. - # - name: Add ICU locale settings - # ansible.builtin.lineinfile: - # path: /etc/postgresql/postgresql.conf - # line: "{{ item }}" - # state: present - # with_items: - # - "lc_collate = 'en-US-x-icu'" - # - "lc_ctype = 'en-US-x-icu'" - # - "icu_locale = 'en-US'" - # when: is_psql_oriole and stage2_nix - # become: yes - - name: Install Git for Nix package management become: yes apt: @@ -321,7 +308,6 @@ line: pgsodium.getkey_script= '{{ pg_bindir }}/pgsodium_getkey.sh' when: stage2_nix -#TODO PR may need to remove this. - name: Create symbolic link for pgsodium_getkey script file: src: "/usr/lib/postgresql/bin/pgsodium_getkey.sh" diff --git a/ansible/vars.yml b/ansible/vars.yml index 98e8764a1..0bcc68ab3 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -3,16 +3,15 @@ ebssurrogate_mode: true async_mode: true postgres_major: - #- "15" - #- "16" + - "15" + - "16" - "orioledb-17" # Full version strings for each major version -# TODO PR uncomment these lines postgres_release: - postgresorioledb-17: "orioledb-17.0.1.000-staging-7" - #postgres15: "15.8.1.005" - #postgres16: "16.3.1.011" + postgresorioledb-17: "orioledb-17.0.1.000" + postgres15: "15.8.1.005" + postgres16: "16.3.1.011" # Non Postgres Extensions pgbouncer_release: "1.19.0" From af742b10ce58fe66855b3882df6ab69cc57ff245 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Nov 2024 13:23:52 -0500 Subject: [PATCH 103/109] fix: try to properly init db in non oriole context --- .github/workflows/test.yml | 28 +--------------------------- ansible/tasks/setup-postgres.yml | 2 +- 2 files changed, 2 insertions(+), 28 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab643cb49..4e1ced281 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,6 +26,7 @@ jobs: build: needs: prepare + if: ${{ !contains(needs.prepare.outputs.postgres_versions, 'orioledb-17') }} strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} @@ -40,33 +41,19 @@ jobs: POSTGRES_PORT: 5478 POSTGRES_PASSWORD: password steps: - - name: Check if orioledb-17 - id: check-version - run: | - if [ "${{ matrix.postgres_version }}" = "\"orioledb-17\"" ]; then - echo "skip=true" >> $GITHUB_OUTPUT - else - echo "skip=false" >> $GITHUB_OUTPUT - fi - - uses: actions/checkout@v3 - if: steps.check-version.outputs.skip != 'true' - uses: DeterminateSystems/nix-installer-action@main - if: steps.check-version.outputs.skip != 'true' - name: Set PostgreSQL version environment variable - if: steps.check-version.outputs.skip != 'true' run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - name: Strip quotes from pg major and set env var - if: steps.check-version.outputs.skip != 'true' run: | stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - name: Generate common-nix.vars.pkr.hcl - if: steps.check-version.outputs.skip != 'true' run: | PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') @@ -74,11 +61,9 @@ jobs: echo "" >> common-nix.vars.pkr.hcl - id: settings - if: steps.check-version.outputs.skip != 'true' run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - name: Generate args - if: steps.check-version.outputs.skip != 'true' id: args run: | ARGS=$(sudo nix run nixpkgs#yq -- 'to_entries | map(select(.value|type == "!!str")) | map(.key + "=" + .value) | join("\n")' ansible/vars.yml) @@ -87,15 +72,12 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - run: docker context create builders - if: steps.check-version.outputs.skip != 'true' - uses: docker/setup-buildx-action@v3 - if: steps.check-version.outputs.skip != 'true' with: endpoint: builders - uses: docker/build-push-action@v5 - if: steps.check-version.outputs.skip != 'true' with: load: true context: . @@ -110,7 +92,6 @@ jobs: cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-${{ steps.settings.outputs.postgres-version }}-${{ matrix.arch }} - name: Start Postgres - if: steps.check-version.outputs.skip != 'true' run: | docker run --rm --pull=never \ -e POSTGRES_PASSWORD=${{ env.POSTGRES_PASSWORD }} \ @@ -119,7 +100,6 @@ jobs: -d supabase/postgres:${{ steps.settings.outputs.postgres-version }} - name: Install psql - if: steps.check-version.outputs.skip != 'true' run: | sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - @@ -127,13 +107,11 @@ jobs: sudo apt install -y --no-install-recommends postgresql-client-${{ env.PGMAJOR }} - name: Install pg_prove - if: steps.check-version.outputs.skip != 'true' run: sudo cpan -T TAP::Parser::SourceHandler::pgTAP env: SHELL: /bin/bash - name: Wait for healthy database - if: steps.check-version.outputs.skip != 'true' run: | count=0 until [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" == "healthy" ]; do @@ -152,7 +130,6 @@ jobs: container: supabase_postgres - name: Run tests - if: steps.check-version.outputs.skip != 'true' run: pg_prove migrations/tests/test.sql env: PGHOST: localhost @@ -162,7 +139,6 @@ jobs: PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - name: Check migrations are idempotent - if: steps.check-version.outputs.skip != 'true' run: | for sql in ./migrations/db/migrations/*.sql; do echo "$0: running $sql" @@ -176,12 +152,10 @@ jobs: PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} - name: Update Dockerfile.dbmate version - if: steps.check-version.outputs.skip != 'true' run: | sed -i 's/%VERSION%/${{ env.PGMAJOR }}/g' migrations/Dockerfile.dbmate - name: verify schema.sql is committed - if: steps.check-version.outputs.skip != 'true' run: | docker compose -f migrations/docker-compose.yaml up db dbmate --abort-on-container-exit if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 69b78c2a7..188a364b8 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -213,7 +213,7 @@ - name: Initialize the database stage2_nix (non-orioledb) become: yes become_user: postgres - shell: source /var/lib/postgresql/.bashrc && initdb -D /var/lib/postgresql/data -o "--allow-group-access" -o "--username=supabase_admin" + shell: source /var/lib/postgresql/.bashrc && /usr/lib/postgresql/bin/pg_ctl -D /var/lib/postgresql/data initdb -o "--allow-group-access" -o "--username=supabase_admin" args: executable: /bin/bash environment: From 25da7c32460b5bd00cfcb926ddd33c47fb5ba0a5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 22 Nov 2024 13:41:57 -0500 Subject: [PATCH 104/109] chore: restore installation of debug and src for all versions` --- ansible/tasks/stage2-setup-postgres.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index f358c956a..d49a0bd70 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -76,17 +76,17 @@ sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supabase-groonga" when: stage2_nix -# - name: Install debug symbols for postgres version -# become: yes -# shell: | -# sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" -# when: stage2_nix +- name: Install debug symbols for postgres version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_debug" + when: stage2_nix -# - name: Install source files for postgresql version -# become: yes -# shell: | -# sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" -# when: stage2_nix +- name: Install source files for postgresql version + become: yes + shell: | + sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#{{postgresql_version}}_src" + when: stage2_nix - name: Remove Git after Nix package installations become: yes From adc0db85ef3b1d9306bca7031dcf9ef69697a6c5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 07:15:28 -0500 Subject: [PATCH 105/109] chore: newline --- docker/nix/build_nix.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index 0b254fd4e..fc9883797 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -24,4 +24,4 @@ nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_16 if [ "$SYSTEM" != "x86_64-darwin" ]; then nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 -fi \ No newline at end of file +fi From cb075b7d1f688aefc8e14754c5f0529e5a6465c5 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 13:31:54 -0500 Subject: [PATCH 106/109] chore: cleaning up and restoring missing code --- .github/workflows/test.yml | 8 ++-- ansible/tasks/stage2-setup-postgres.yml | 48 ++++++++++++++++++- .../20241031003909_create_orioledb.sql | 2 +- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7b37aac77..487aeadff 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,7 +30,6 @@ jobs: build: needs: prepare - if: ${{ !contains(needs.prepare.outputs.postgres_versions, 'orioledb-17') }} strategy: matrix: postgres_version: ${{ fromJson(needs.prepare.outputs.postgres_versions) }} @@ -40,6 +39,7 @@ jobs: - runner: ubuntu-22.04 arch: arm64 runs-on: ${{ matrix.runner }} + if: ${{ matrix.postgres_version != 'orioledb-17' }} timeout-minutes: 180 env: POSTGRES_PORT: 5478 @@ -53,7 +53,6 @@ jobs: substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV @@ -64,13 +63,14 @@ jobs: - name: Generate common-nix.vars.pkr.hcl run: | - PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - id: settings + # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - name: Generate args @@ -84,7 +84,7 @@ jobs: - name: verify schema.sql is committed run: | GIT_SHA=${{github.sha}} - nix run github:supabase/postgres/${GIT_SHA}#dbmate-tool -- --version ${{ env.PGMAJOR }} + nix run github:supabase/postgres/develop#dbmate-tool -- --version ${{ env.PGMAJOR }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" git diff migrations/schema-${{ env.PGMAJOR }}.sql diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index f32600348..c3ce46941 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -4,6 +4,52 @@ # sudo -u postgres bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install nixpkgs#openjdk11" # It was decided to leave pljava disabled at https://github.com/supabase/postgres/pull/690 therefore removing this task +- name: Check psql_version and modify supautils.conf and postgresql.conf if necessary + block: + - name: Check if psql_version is psql_orioledb-16 + set_fact: + is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}" + + - name: Remove specified extensions from postgresql.conf if oriole-16 build + ansible.builtin.command: + cmd: > + sed -i 's/ timescaledb,//g' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Remove specified extensions from supautils.conf if oriole-16 build + ansible.builtin.command: + cmd: > + sed -i 's/ timescaledb,//g; s/ vector,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' + /etc/postgresql-custom/supautils.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Remove db_user_namespace from postgresql.conf if oriole-xx build + ansible.builtin.command: + cmd: > + sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Append orioledb to shared_preload_libraries append within closing quote + ansible.builtin.command: + cmd: > + sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' + /etc/postgresql/postgresql.conf + when: is_psql_oriole and stage2_nix + become: yes + + - name: Add default_table_access_method setting + ansible.builtin.lineinfile: + path: /etc/postgresql/postgresql.conf + line: "default_table_access_method = 'orioledb'" + state: present + when: is_psql_oriole and stage2_nix + become: yes + - name: Install Postgres from nix binary cache become: yes shell: | @@ -198,7 +244,7 @@ recurse: yes when: stage2_nix -- name: Check psql_version and run postgis linking if not oriole-16 +- name: Check psql_version and run postgis linking if not oriole-xx block: - name: Check if psql_version is psql_orioledb-17 set_fact: diff --git a/migrations/db/migrations/20241031003909_create_orioledb.sql b/migrations/db/migrations/20241031003909_create_orioledb.sql index 29bd0ae5f..dbfe5a63e 100644 --- a/migrations/db/migrations/20241031003909_create_orioledb.sql +++ b/migrations/db/migrations/20241031003909_create_orioledb.sql @@ -8,4 +8,4 @@ begin end $$; -- migrate:down -drop extension if exists orioledb; \ No newline at end of file +drop extension if exists orioledb; From 9260c021b4ad8b283f10afd0459a45aad63ac472 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 13:39:33 -0500 Subject: [PATCH 107/109] chore: cleanup --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ae375b82a..45464e3f0 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,4 @@ result* .idea/ .vscode/ -db \ No newline at end of file +db From cdbc06fbee2ffc38300e2ccd92f0f4da63ba0725 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 15:23:02 -0500 Subject: [PATCH 108/109] chore: fix gh action conditonal for oriole --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 487aeadff..e1f6c4f94 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,7 +39,7 @@ jobs: - runner: ubuntu-22.04 arch: arm64 runs-on: ${{ matrix.runner }} - if: ${{ matrix.postgres_version != 'orioledb-17' }} + if: matrix.postgres_version != 'orioledb-17' timeout-minutes: 180 env: POSTGRES_PORT: 5478 From b04925c96d735796a3697709f05a96b813b0ae32 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 25 Nov 2024 15:29:34 -0500 Subject: [PATCH 109/109] fix: filter orioledb-17 from test on read --- .github/workflows/test.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e1f6c4f94..1b60cfcca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,12 +1,10 @@ name: Test Database - on: push: branches: - develop pull_request: workflow_dispatch: - jobs: prepare: runs-on: ubuntu-latest @@ -21,13 +19,11 @@ jobs: extra-conf: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Set PostgreSQL versions id: set-versions run: | - VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') + VERSIONS=$(nix run nixpkgs#yq -- '.postgres_major[] | select(. != "orioledb-17")' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]') echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT - build: needs: prepare strategy: @@ -39,40 +35,31 @@ jobs: - runner: ubuntu-22.04 arch: arm64 runs-on: ${{ matrix.runner }} - if: matrix.postgres_version != 'orioledb-17' timeout-minutes: 180 env: POSTGRES_PORT: 5478 POSTGRES_PASSWORD: password steps: - uses: actions/checkout@v4 - - uses: DeterminateSystems/nix-installer-action@main with: extra-conf: | substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI=% cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - name: Set PostgreSQL version environment variable run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.postgres_version }}" >> $GITHUB_ENV - - name: Strip quotes from pg major and set env var run: | stripped_version=$(echo ${{ matrix.postgres_version }} | sed 's/^"\(.*\)"$/\1/') echo "PGMAJOR=$stripped_version" >> $GITHUB_ENV - - name: Generate common-nix.vars.pkr.hcl run: | PG_VERSION=$(nix run nixpkgs#yq -- '.postgres_release["postgres'${{ matrix.postgres_version }}'"]' ansible/vars.yml) PG_VERSION=$(echo $PG_VERSION | tr -d '"') # Remove any surrounding quotes echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl - # Ensure there's a newline at the end of the file echo "" >> common-nix.vars.pkr.hcl - - id: settings - # Remove spaces and quotes to get the raw version string run: sed -r 's/(\s|\")+//g' common-nix.vars.pkr.hcl >> $GITHUB_OUTPUT - - name: Generate args id: args run: | @@ -80,7 +67,6 @@ jobs: echo "result<> $GITHUB_OUTPUT echo "$ARGS" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - - name: verify schema.sql is committed run: | GIT_SHA=${{github.sha}}