pgBackRest AWS S3 configuration example

[rez0n]

Hi there,
I want to setup WAL's archiving to AWS S3 to make reserve (?) place where replicas can get WAL's in case of primary failure. I hope my thoughts are correct (reference in issue comments).

I not found any example how to configure pgBackRest (with S3) using this playbook in correct way, referring to the pgBackRest documentation I modified vars in this way, please let me know if I something missed.

postgresql_parameters:
  ...
  - { option: "archive_mode", value: "on" }
  - { option: "archive_timeout", value: "1800s" } # why so much?
  - { option: "archive_command", value: "pgbackrest --stanza={{ pgbackrest_stanza }} archive-push %p" }

pgbackrest_install: true 
pgbackrest_install_from_pgdg_repo: true
pgbackrest_stanza: "main" 
pgbackrest_repo_type: "s3"  
pgbackrest_repo_host: "10.128.64.50"
pgbackrest_repo_user: "postgres"
pgbackrest_conf_file: "/etc/pgbackrest/pgbackrest.conf"

pgbackrest_conf:
  global:  # [global] section
    - { option: "log-level-file", value: "detail" }
    - { option: "log-path", value: "/var/log/pgbackrest" }
    - { option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}" }
    - { option: "repo1-host", value: "{{ pgbackrest_repo_host }}" }
    - { option: "repo1-host-user", value: "{{ pgbackrest_repo_user }}" }
   
    - { option: "repo1-retention-full", value: 3 }
    - { option: "repo1-s3-bucket", value: "pgcluster_pgbackrest" }
    - { option: "repo1-repo1-path", value: "pgbackrest_data" }
    - { option: "repo1-s3-region", value: "us-east-1" }
    - { option: "repo1-s3-endpoint", value: "s3.amazonaws.com" }
    - { option: "repo1-s3-key", value: "AWS_ACCESS_KEY" }
    - { option: "repo1-s3-key-secret", value: "AWS_SECRET_KEY" }

  stanza:  # [stanza_name] section
    - { option: "pg1-path", value: "{{ postgresql_data_dir }}" }
    - { option: "process-max", value: "2" }
    - { option: "recovery-option", value: "recovery_target_action=promote" }
pgbackrest_patroni_cluster_restore_command:
  '/usr/bin/pgbackrest --stanza={{ pgbackrest_stanza }} --delta restore'
pgbackrest_cron: 
  - { name: "Full Backup", type: "full", hour: "06", minute: "30", dow: "0" }
  - { name: "Diff Backup", type: "diff", hour: "06", minute: "30", dow: "1-6" }

Thanks.

[vitabaks]

@rez0n Hi!

the variables repo1-host and repo1-host-user are superfluous since you have chosen pgbackrest_repo_type: "s3".

The pgbackrest documentation is an excellent source of information on setting up pgbackrest.conf and I strongly recommend that you study the documentation before using it.

I don't personally use amazon s3 but I use minio s3, here is an example of one of my configurations:

pgbackrest_install: true
pgbackrest_install_from_pgdg_repo: true
pgbackrest_stanza: "my_stanza_name"
pgbackrest_repo_type: "s3"
pgbackrest_repo_host: ""
pgbackrest_repo_user: ""
pgbackrest_conf_file: "/etc/pgbackrest/pgbackrest.conf"

pgbackrest_conf:
  global:  # [global] section
    - {option: "repo1-type", value: "{{ pgbackrest_repo_type |lower }}"}
    - {option: "repo1-path", value: "/repo"}
    - {option: "repo1-s3-endpoint", value: "https://minio.mydomain.com"}
    - {option: "repo1-s3-uri-style", value: "path"}
    - {option: "repo1-s3-bucket", value: "pgbackrest"}
    - {option: "repo1-s3-verify-tls", value: "n"}
    - {option: "repo1-s3-key", value: "{{ ott_repo_s3_key_vault }}"}
    - {option: "repo1-s3-key-secret", value: "{{ ott_repo_s3_key_secret_vault }}"}
    - {option: "repo1-s3-region", value: "eu-west-3"}
    - {option: "repo1-retention-full", value: "4"}
    - {option: "repo1-retention-archive", value: "2"}
    - {option: "repo1-retention-archive-type", value: "full"}
    - {option: "start-fast", value: "y"}
    - {option: "delta", value: "y"}
    - {option: "archive-check", value: "y"}
    - {option: "archive-copy", value: "y"}
    - {option: "archive-timeout", value: "180"}
    - {option: "stop-auto", value: "y"}
    - {option: "resume", value: "n"}
    - {option: "link-all", value: "y"}
    - {option: "compress-type", value: "lz4"}
    - {option: "log-level-file", value: "detail"}
    - {option: "log-path", value: "/var/log/pgbackrest"}
    - {option: "process-max", value: "2"}
  stanza:  # [stanza_name] section
    - {option: "pg1-path", value: "{{ postgresql_data_dir }}"}
    - {option: "recovery-option", value: "recovery_target_action=promote"}
    - {option: "process-max", value: "6"}
    - {option: "log-level-console", value: "info"}

Also, note that you will need to execute the pgbackrest --stanza=<my_stanza_name> stanza-create command yourself since it is not automated yet (but I think we will add it soon - issue)

[rez0n]

Much thanks for the example!