Merge branch 'develop' into etienne/sec-197-use-nonewpriviliges-for-postgres

Author: staaldraad

Diff for: .github/CODEOWNERS

@@ -1,4 +1,4 @@
 * @supabase/backend @supabase/postgres
-migrations/ @supabase/cli @supabase/backend
+migrations/ @supabase/dev-workflows @supabase/postgres @supabase/backend
 docker/orioledb @supabase/postgres @supabase/backend
 common.vars.pkr.hcl @supabase/postgres @supabase/backend

Diff for: .github/workflows/qemu-image-build.yml

@@ -23,10 +23,10 @@ jobs:
 
       - uses: DeterminateSystems/nix-installer-action@main
 
-      - name: Set PostgreSQL versions - only builds pg15 atm
+      - name: Set PostgreSQL versions - only builds pg17 atm
         id: set-versions
         run: |
-          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[0]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
+          VERSIONS=$(nix run nixpkgs#yq --  '.postgres_major[1]' ansible/vars.yml | nix run nixpkgs#jq -- -R -s -c 'split("\n")[:-1]')
           echo "postgres_versions=$VERSIONS" >> $GITHUB_OUTPUT
 
   build:

Diff for: README.md

@@ -81,7 +81,7 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 
 | Goodie | Version | Description |
 | ------------- | :-------------: | ------------- |
-| [PgBouncer](https://www.pgbouncer.org/) | [1.16.1](http://www.pgbouncer.org/changelog.html#pgbouncer-116x) | Set up Connection Pooling. |
+| [PgBouncer](https://www.pgbouncer.org/) | [1.19.0](http://www.pgbouncer.org/changelog.html#pgbouncer-119x) | Set up Connection Pooling. |
 | [PostgREST](https://postgrest.org/en/stable/) | [v12.2.3](https://github.com/PostgREST/postgrest/releases/tag/v12.2.3) | Instantly transform your database into an RESTful API. |
 | [WAL-G](https://github.com/wal-g/wal-g#wal-g) | [v2.0.1](https://github.com/wal-g/wal-g/releases/tag/v2.0.1) | Tool for physical database backup and recovery. | -->
 
@@ -126,4 +126,4 @@ TODO: find way to automate this
 
 We are building the features of Firebase using enterprise-grade, open source products. We support existing communities wherever possible, and if the products don’t exist we build them and open source them ourselves.
 
-[![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase)
+[![New Sponsor](https://user-images.githubusercontent.com/10214025/90518111-e74bbb00-e198-11ea-8f88-c9e3c1aa4b5b.png)](https://github.com/sponsors/supabase)

Diff for: amazon-arm64-nix.pkr.hcl

@@ -228,11 +228,6 @@ build {
     destination = "/tmp"
   }
 
-  provisioner "file" {
-    source = "ebssurrogate/files/unit-tests"
-    destination = "/tmp"
-  }
-
   # Copy ansible playbook
   provisioner "shell" {
     inline = ["mkdir /tmp/ansible-playbook"]

Diff for: ansible/files/gotrue-optimizations.service.j2

@@ -5,6 +5,7 @@ Description=GoTrue (Auth) optimizations
 Type=oneshot
 # we don't want failures from this command to cause PG startup to fail
 ExecStart=/bin/bash -c "/opt/supabase-admin-api optimize auth --destination-config-file-path /etc/gotrue/gotrue.generated.env ; exit 0"
+ExecStartPost=/bin/bash -c "cp -a /etc/gotrue/gotrue.generated.env /etc/auth.d/20_generated.env ; exit 0"
 User=postgrest
 
 [Install]

Diff for: ansible/files/gotrue.service.j2

@@ -4,7 +4,7 @@ Description=Gotrue
 [Service]
 Type=simple
 WorkingDirectory=/opt/gotrue
-ExecStart=/opt/gotrue/gotrue
+ExecStart=/opt/gotrue/gotrue --config-dir /etc/auth.d
 User=gotrue
 Restart=always
 RestartSec=3

Diff for: ansible/tasks/setup-gotrue.yml

@@ -30,6 +30,13 @@
     owner: gotrue
     mode: 0775
 
+- name: gotrue - create /etc/auth.d
+  file:
+    path: /etc/auth.d
+    state: directory
+    owner: gotrue
+    mode: 0755
+
 - name: gotrue - unpack archive in /opt/gotrue
   unarchive:
     remote_src: yes

Diff for: ansible/tasks/setup-wal-g.yml

@@ -44,7 +44,7 @@
 
 - name: Create symlink to make wal-g-v2 the default wal-g
   ansible.builtin.file:
-    src: /usr/local/bin/wal-g-v2
+    src: /home/wal-g/.nix-profile/bin/wal-g-2
     dest: /usr/local/bin/wal-g
     state: link
     force: yes

Diff for: ansible/tasks/test-image.yml

@@ -1,9 +1,3 @@
-- name: install pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-  when: debpkg_mode
-
 # - name: Temporarily disable PG Sodium references in config
 #   become: yes
 #   become_user: postgres
@@ -16,9 +10,9 @@
   become_user: postgres
   shell:
     cmd: >
-      sed -i.bak 
-      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/' 
-      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/' 
+      sed -i.bak
+      -e 's/\(shared_preload_libraries = '\''.*\)pgsodium,\(.*'\''\)/\1\2/'
+      -e 's/\(shared_preload_libraries = '\''.*\)supabase_vault,\(.*'\''\)/\1\2/'
       -e 's/\(shared_preload_libraries = '\''.*\), *supabase_vault'\''/\1'\''/'
       -e 's/pgsodium.getkey_script=/#pgsodium.getkey_script=/'
       /etc/postgresql/postgresql.conf
@@ -74,53 +68,6 @@
     LOCALE_ARCHIVE: /usr/lib/locale/locale-archive
   when: stage2_nix
 
-
-- name: Check psql_version and modify migrations if oriole-xx
-  block:
-    - name: Check if psql_version is psql_orioledb-xx
-      set_fact:
-        is_psql_oriole: "{{ psql_version in ['psql_orioledb-16', 'psql_orioledb-17'] }}"
-
-    - name: Remove specified extensions from SQL file
-      ansible.builtin.command:
-        cmd: >
-          sed -i '/\\ir.*\(timescaledb\|postgis\|pgrouting\|plv8\).*\.sql/d' /tmp/migrations/tests/extensions/test.sql
-      when: is_psql_oriole
-      become: yes
-
-    - name: Remove specified extension files from extensions directory
-      ansible.builtin.find:
-        paths: /tmp/migrations/tests/extensions
-        patterns: 
-          - '*timescaledb*.sql'
-          - '*plv8*.sql'
-          - '*postgis*.sql'
-          - '*pgrouting*.sql'
-      register: files_to_remove
-      when: is_psql_oriole
-
-    - name: Delete matched extension files
-      ansible.builtin.file:
-        path: "{{ item.path }}"
-        state: absent
-      loop: "{{ files_to_remove.files }}"
-      when: is_psql_oriole
-      become: yes
-
-- name: Run Unit tests (with filename unit-test-*) on Postgres Database
-  shell: /usr/bin/pg_prove -U postgres -h localhost -d postgres -v /tmp/unit-tests/unit-test-*.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-
-- name: Run migrations tests
-  shell: /usr/bin/pg_prove -U supabase_admin -h localhost -d postgres -v tests/test.sql
-  register: retval
-  failed_when: retval.rc != 0
-  when: debpkg_mode or stage2_nix
-  args:
-    chdir: /tmp/migrations
-
 - name: Re-enable PG Sodium references in config
   become: yes
   become_user: postgres
@@ -132,14 +79,6 @@
   shell: /usr/lib/postgresql/bin/psql --no-password --no-psqlrc -d postgres -h localhost -U supabase_admin -c 'SELECT pg_stat_statements_reset(); SELECT pg_stat_reset();'
   when: debpkg_mode or stage2_nix
 
-- name: remove pg_prove
-  apt:
-    pkg:
-      - libtap-parser-sourcehandler-pgtap-perl
-    state: absent
-    autoremove: yes
-  when: debpkg_mode
-
 - name: Stop Postgres Database
   become: yes
   become_user: postgres

Diff for: ansible/vars.yml

@@ -8,8 +8,8 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.0.1.052-orioledb.etcro"
-  postgres15: "15.8.1.059-rc.etcro"
+  postgresorioledb-17: "17.0.1.054-orioledb"
+  postgres15: "15.8.1.061"
 
 # Non Postgres Extensions
 pgbouncer_release: "1.19.0"
@@ -51,7 +51,7 @@ postgres_exporter_release_checksum:
   arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0
   amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84
 
-adminapi_release: 0.75.0
+adminapi_release: 0.76.0
 adminmgr_release: 0.24.1
 
 vector_x86_deb: "https://packages.timber.io/vector/0.22.3/vector_0.22.3-1_amd64.deb"

Diff for: ebssurrogate/files/unit-tests/unit-test-01.sql

@@ -180,9 +180,6 @@ function setup_chroot_environment {
 	# Copy migrations
 	cp -r /tmp/migrations /mnt/tmp/
 
-	# Copy unit tests 
-	cp -r /tmp/unit-tests /mnt/tmp/
-
 	# Copy the bootstrap script into place and execute inside chroot
 	cp /tmp/chroot-bootstrap-nix.sh /mnt/tmp/chroot-bootstrap-nix.sh
 	chroot /mnt /tmp/chroot-bootstrap-nix.sh

Diff for: ebssurrogate/scripts/surrogate-bootstrap-nix.sh

@@ -822,12 +822,15 @@
                 exit 1
               fi
 
+              echo "Running migrations tests"
+              pg_prove -p 5435 -U supabase_admin -h localhost -d postgres -v ${./migrations/tests}/test.sql
+
               # Copy logs to output
               for logfile in $(find /tmp -name postgresql.log -type f); do
                 cp "$logfile" $out/postgresql.log
               done
               exit 0
-            '';      
+            '';
     in
       rec {
         # The list of all packages that can be built with 'nix build'. The list
@@ -865,6 +868,7 @@
             local-infra-bootstrap = mkApp "local-infra-bootstrap" "local-infra-bootstrap";
             dbmate-tool = mkApp "dbmate-tool" "dbmate-tool";
             update-readme = mkApp "update-readme" "update-readme";
+            show-commands = mkApp "show-commands" "show-commands";
           };
 
         # 'devShells.default' lists the set of packages that are included in the

Diff for: flake.nix

@@ -42,8 +42,6 @@ nix run github:supabase/postgres/mybranch#dbmate-tool -- --version 15
 
 aiming to provide a single source of truth for migrations on the platform that can be depended upon by those components. For more information on goals see [the RFC](https://www.notion.so/supabase/Centralize-SQL-Migrations-cd3847ae027d4f2bba9defb2cc82f69a)
 
-
-
 ## How it was Created
 
 Migrations were pulled (in order) from:
@@ -53,9 +51,8 @@ Migrations were pulled (in order) from:
 
 For compatibility with hosted projects, we include [migrate.sh](migrate.sh) that executes migrations in the same order as ami build:
 
-1. Run all `db/init-scripts` with `postgres` superuser role.
-2. Run all `db/migrations` with `supabase_admin` superuser role.
-3. Finalize role passwords with `/etc/postgresql.schema.sql` if present.
+1. Run all `db/migrations` with `supabase_admin` superuser role.
+2. Finalize role passwords with `/etc/postgresql.schema.sql` if present.
 
 Additionally, [supabase/postgres](https://github.com/supabase/postgres/blob/develop/ansible/playbook-docker.yml#L9) image contains several migration scripts to configure default extensions. These are run first by docker entrypoint and included in ami by ansible.
 

Diff for: migrations/README.md

@@ -38,11 +38,6 @@ begin
   end if;
 end \$\$
 EOSQL
-    # run init scripts as postgres user
-    for sql in "$db"/init-scripts/*.sql; do
-        echo "$0: running $sql"
-        psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -f "$sql"
-    done
     psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
     # run migrations as super user - postgres user demoted in post-setup
     for sql in "$db"/migrations/*.sql; do
@@ -54,8 +49,6 @@ else
   create role postgres superuser login password '$PGPASSWORD';
   alter database postgres owner to postgres;
 EOSQL
-    # run init scripts as postgres user
-    DBMATE_MIGRATIONS_DIR="$db/init-scripts" DATABASE_URL="postgres://postgres:$connect" dbmate --no-dump-schema migrate
     psql -v ON_ERROR_STOP=1 --no-password --no-psqlrc -U postgres -c "ALTER USER supabase_admin WITH PASSWORD '$PGPASSWORD'"
     # run migrations as super user - postgres user demoted in post-setup
     DBMATE_MIGRATIONS_DIR="$db/migrations" DATABASE_URL="postgres://supabase_admin:$connect" dbmate --no-dump-schema migrate