From 31037363c2b73aeab9632b8eddec78e11fa82ab0 Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Wed, 24 Jul 2024 09:29:36 +0100 Subject: [PATCH 01/37] Adding support for legacy compilers (#448) * Adding support for legacy compilers * Adding inventoryfile to test upgrade legacy workflow * feat(documentation): add instructions for converting compilers to legacy This commit introduces a new section in the `convert.md` documentation. It provides instructions on how to convert compilers to legacy compilers for Puppet Enterprise installations using puppetlabs-peadm version 3.21 or later, as well as for versions prior to 3.21. The new section includes specific commands to run and references to other relevant documentation. * Fixing lint and regenerating referencemd * PE-38772 Node groups added for legacy compilers (#455) * Fixing typo for parameter in docs --------- Co-authored-by: Ioannis Karasavvaidis Co-authored-by: Neil Anderson Co-authored-by: Aaron Shannon --- .github/workflows/test-upgrade-legacy.yaml | 163 +++++ REFERENCE.md | 612 +++++++----------- documentation/convert.md | 20 +- .../upgrade_with_legacy_compilers.md | 45 ++ functions/oid.pp | 1 + manifests/setup/legacy_compiler_group.pp | 50 ++ manifests/setup/node_manager.pp | 50 ++ plans/convert_compiler_to_legacy.pp | 58 ++ plans/subplans/component_install.pp | 4 + plans/subplans/install.pp | 2 + plans/update_compiler_extensions.pp | 25 + plans/upgrade.pp | 18 +- .../plans/provision_test_cluster.pp | 3 + 13 files changed, 657 insertions(+), 394 deletions(-) create mode 100644 .github/workflows/test-upgrade-legacy.yaml create mode 100644 documentation/upgrade_with_legacy_compilers.md create mode 100644 manifests/setup/legacy_compiler_group.pp create mode 100644 plans/convert_compiler_to_legacy.pp create mode 100644 plans/update_compiler_extensions.pp diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml new file mode 100644 index 00000000..a8fb954a --- /dev/null +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -0,0 +1,163 @@ +--- +name: "Upgrade PE with one legacy compiler" + +on: + pull_request: + paths: + - ".github/workflows/**/*" + - "spec/**/*" + - "lib/**/*" + - "tasks/**/*" + - "functions/**/*" + - "types/**/*" + - "plans/**/*" + - "hiera/**/*" + - "manifests/**/*" + - "templates/**/*" + - "files/**/*" + - "metadata.json" + - "Rakefile" + - "Gemfile" + - "provision.yaml" + - ".rspec" + - ".rubocop.yml" + - ".puppet-lint.rc" + - ".fixtures.yml" + branches: [main] + workflow_dispatch: + ssh-debugging: + description: "Boolean; whether or not to pause for ssh debugging" + required: true + default: "false" + +jobs: + test-install: + name: "PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}" + runs-on: ubuntu-20.04 + env: + BOLT_GEM: true + BOLT_DISABLE_ANALYTICS: true + LANG: "en_US.UTF-8" + strategy: + fail-fast: false + matrix: + architecture: + - "large-with-two-compilers" + image: + - "almalinux-cloud/almalinux-8" + version: + - "2023.6.0" + to_version: + - "2023.7.0" + + steps: + - name: "Start SSH session" + if: ${{ github.event.inputs.ssh-debugging == 'true' }} + uses: luchihoratiu/debug-via-ssh@main + with: + NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} + SSH_PASS: ${{ secrets.SSH_PASS }} + + - name: "Checkout Source" + uses: actions/checkout@v2 + + - name: "Activate Ruby 2.7" + uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.7" + bundler-cache: true + + - name: "Print bundle environment" + if: ${{ github.repository_owner == 'puppetlabs' }} + run: | + echo ::group::info:bundler + bundle env + echo ::endgroup:: + + - name: "Provision test cluster" + timeout-minutes: 15 + run: | + echo ::group::prepare + mkdir -p $HOME/.ssh + echo 'Host *' > $HOME/.ssh/config + echo ' ServerAliveInterval 150' >> $HOME/.ssh/config + echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config + bundle exec rake spec_prep + echo ::endgroup:: + + echo ::group::provision + bundle exec bolt plan run peadm_spec::provision_test_cluster \ + --modulepath spec/fixtures/modules \ + provider=provision_service \ + image=${{ matrix.image }} \ + architecture=${{ matrix.architecture }} + echo ::endgroup:: + + echo ::group::info:request + cat request.json || true; echo + echo ::endgroup:: + + echo ::group::info:inventory + sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true + echo ::endgroup:: + + - name: Set up yq + uses: frenck/action-setup-yq@v1 + with: + version: v4.30.5 + + - name: 'Install PE on test cluster' + timeout-minutes: 120 + run: | + bundle exec bolt plan run peadm_spec::install_test_cluster \ + --inventoryfile spec/fixtures/litmus_inventory.yaml \ + --modulepath spec/fixtures/modules \ + architecture="large" \ + version=${{ matrix.version }} + + - name: 'Wait as long as the file ${HOME}/pause file is present' + if: ${{ always() && github.event.inputs.ssh-debugging == 'true' }} + run: | + while [ -f "${HOME}/pause" ] ; do + echo "${HOME}/pause present, sleeping for 60 seconds..." + sleep 60 + done + echo "${HOME}/pause absent, continuing workflow." + + - name: 'Convert one compiler to legacy' + timeout-minutes: 120 + run: | + primary=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .uri' spec/fixtures/litmus_inventory.yaml) + compiler=$(yq '.groups[].targets[] | select(.vars.role == "compiler") | .uri' spec/fixtures/litmus_inventory.yaml | head -n 1) + + bundle exec bolt plan run peadm::convert_compiler_to_legacy \ + --inventoryfile spec/fixtures/litmus_inventory.yaml \ + --modulepath spec/fixtures/modules \ + --no-host-key-check \ + primary_host=$primary \ + legacy_hosts=$compiler + + + - name: 'Upgrade PE on test cluster' + timeout-minutes: 120 + run: | + bundle exec bolt plan run peadm_spec::upgrade_test_cluster \ + --inventoryfile spec/fixtures/litmus_inventory.yaml \ + --modulepath spec/fixtures/modules \ + --no-host-key-check \ + architecture="large" \ + version=${{ matrix.to_version }} + + - name: "Tear down test cluster" + if: ${{ always() }} + continue-on-error: true + run: | + if [ -f spec/fixtures/litmus_inventory.yaml ]; then + echo ::group::tear_down + bundle exec rake 'litmus:tear_down' + echo ::endgroup:: + + echo ::group::info:request + cat request.json || true; echo + echo ::endgroup:: + fi diff --git a/REFERENCE.md b/REFERENCE.md index 065b6d75..8d7c910e 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -8,123 +8,130 @@ #### Private Classes -* `peadm::setup::convert_node_manager`: Used during the peadm::convert plan -* `peadm::setup::convert_pre20197`: Defines configuration needed for converting PE 2018 -* `peadm::setup::node_manager`: Configures PEAdm's required node groups -* `peadm::setup::node_manager_yaml`: Set up the node_manager.yaml file in the temporary Bolt confdir +- `peadm::setup::convert_node_manager`: Used during the peadm::convert plan +- `peadm::setup::convert_pre20197`: Defines configuration needed for converting PE 2018 +- `peadm::setup::legacy_compiler_group` +- `peadm::setup::node_manager`: Configures PEAdm's required node groups +- `peadm::setup::node_manager_yaml`: Set up the node_manager.yaml file in the temporary Bolt confdir ### Functions -* [`peadm::assert_supported_architecture`](#peadm--assert_supported_architecture): Assert that the architecture given is a supported one -* [`peadm::assert_supported_bolt_version`](#peadm--assert_supported_bolt_version): Assert that the Bolt executable running PEAdm is a supported version -* [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm -* [`peadm::bolt_version`](#peadm--bolt_version) -* [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input -* [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts -* [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash -* [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors -* [`peadm::determine_status`](#peadm--determine_status): Produces a summarized hash of the given status data -* [`peadm::fail_on_transport`](#peadm--fail_on_transport): Fails if any nodes have the chosen transport. Useful for excluding PCP when it's not appopriate -* [`peadm::file_content_upload`](#peadm--file_content_upload) -* [`peadm::file_or_content`](#peadm--file_or_content) -* [`peadm::flatten_compact`](#peadm--flatten_compact) -* [`peadm::generate_pe_conf`](#peadm--generate_pe_conf): Generate a pe.conf file in JSON format -* [`peadm::get_pe_conf`](#peadm--get_pe_conf) -* [`peadm::get_targets`](#peadm--get_targets): Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - It returns an Array[Target -* [`peadm::migration_opts_default`](#peadm--migration_opts_default) -* [`peadm::node_manager_yaml_location`](#peadm--node_manager_yaml_location) -* [`peadm::oid`](#peadm--oid) -* [`peadm::plan_step`](#peadm--plan_step) -* [`peadm::recovery_opts_all`](#peadm--recovery_opts_all) -* [`peadm::recovery_opts_default`](#peadm--recovery_opts_default) -* [`peadm::update_pe_conf`](#peadm--update_pe_conf): Update the pe.conf file on a target with the provided hash -* [`peadm::wait_until_service_ready`](#peadm--wait_until_service_ready): A convenience function to help remember port numbers for services and handle running the wait_until_service_ready task +- [`peadm::assert_supported_architecture`](#peadm--assert_supported_architecture): Assert that the architecture given is a supported one +- [`peadm::assert_supported_bolt_version`](#peadm--assert_supported_bolt_version): Assert that the Bolt executable running PEAdm is a supported version +- [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm +- [`peadm::bolt_version`](#peadm--bolt_version) +- [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input +- [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts +- [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash +- [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors +- [`peadm::determine_status`](#peadm--determine_status): Produces a summarized hash of the given status data +- [`peadm::fail_on_transport`](#peadm--fail_on_transport): Fails if any nodes have the chosen transport. Useful for excluding PCP when it's not appopriate +- [`peadm::file_content_upload`](#peadm--file_content_upload) +- [`peadm::file_or_content`](#peadm--file_or_content) +- [`peadm::flatten_compact`](#peadm--flatten_compact) +- [`peadm::generate_pe_conf`](#peadm--generate_pe_conf): Generate a pe.conf file in JSON format +- [`peadm::get_pe_conf`](#peadm--get_pe_conf) +- [`peadm::get_targets`](#peadm--get_targets): Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - It returns an Array[Target +- [`peadm::migration_opts_default`](#peadm--migration_opts_default) +- [`peadm::node_manager_yaml_location`](#peadm--node_manager_yaml_location) +- [`peadm::oid`](#peadm--oid) +- [`peadm::plan_step`](#peadm--plan_step) +- [`peadm::recovery_opts_all`](#peadm--recovery_opts_all) +- [`peadm::recovery_opts_default`](#peadm--recovery_opts_default) +- [`peadm::update_pe_conf`](#peadm--update_pe_conf): Update the pe.conf file on a target with the provided hash +- [`peadm::wait_until_service_ready`](#peadm--wait_until_service_ready): A convenience function to help remember port numbers for services and handle running the wait_until_service_ready task ### Data types -* [`Peadm::Known_hosts`](#Peadm--Known_hosts) -* [`Peadm::Ldap_config`](#Peadm--Ldap_config) -* [`Peadm::Pe_version`](#Peadm--Pe_version) -* [`Peadm::Pem`](#Peadm--Pem) -* [`Peadm::Recovery_opts`](#Peadm--Recovery_opts) -* [`Peadm::SingleTargetSpec`](#Peadm--SingleTargetSpec): A SingleTargetSpec represents any String, Target or single-element array of one or the other that can be passed to get_targets() to return an +- [`Peadm::Known_hosts`](#Peadm--Known_hosts) +- [`Peadm::Ldap_config`](#Peadm--Ldap_config) +- [`Peadm::Pe_version`](#Peadm--Pe_version) +- [`Peadm::Pem`](#Peadm--Pem) +- [`Peadm::Recovery_opts`](#Peadm--Recovery_opts) +- [`Peadm::SingleTargetSpec`](#Peadm--SingleTargetSpec): A SingleTargetSpec represents any String, Target or single-element array of one or the other that can be passed to get_targets() to return an ### Tasks -* [`agent_install`](#agent_install): Install the Puppet agent from a master -* [`backup_classification`](#backup_classification): A task to call the classification api and write to file -* [`cert_data`](#cert_data): Return certificate data related to the Puppet agent -* [`cert_valid_status`](#cert_valid_status): Check primary for valid state of a certificate -* [`code_manager`](#code_manager): Perform various code manager actions -* [`code_sync_status`](#code_sync_status): A task to confirm code is in sync accross the cluster for clusters with code manager configured -* [`divert_code_manager`](#divert_code_manager): Divert the code manager live-dir setting -* [`download`](#download): Download a file using curl -* [`enable_replica`](#enable_replica): Execute the enable replica puppet command -* [`filesize`](#filesize): Return the size of a file in bytes -* [`get_peadm_config`](#get_peadm_config): Run on a PE primary node to return the currently configured PEAdm parameters -* [`get_psql_version`](#get_psql_version): Run on a PE PSQL node to return the major version of the PSQL server currently installed -* [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output -* [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location -* [`mv`](#mv): Wrapper task for mv command -* [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string -* [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball -* [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console -* [`pe_uninstall`](#pe_uninstall): Uninstall Puppet Enterprise -* [`precheck`](#precheck): Return pre-check information about a system -* [`provision_replica`](#provision_replica): Execute the replica provision puppet command -* [`puppet_infra_upgrade`](#puppet_infra_upgrade): Execute the puppet infra upgrade command -* [`puppet_runonce`](#puppet_runonce): Run the Puppet agent one time -* [`rbac_token`](#rbac_token): Get and save an rbac token for the root user, admin rbac user -* [`read_file`](#read_file): Read the contents of a file -* [`reinstall_pe`](#reinstall_pe): Reinstall PE, only to be used to restore PE -* [`restore_classification`](#restore_classification): A short description of this task -* [`sign_csr`](#sign_csr): Submit a certificate signing request -* [`ssl_clean`](#ssl_clean): Clean an agent's certificate -* [`submit_csr`](#submit_csr): Submit a certificate signing request -* [`transform_classification_groups`](#transform_classification_groups): Transform the user groups from a source backup to a list of groups on the target server -* [`wait_until_service_ready`](#wait_until_service_ready): Return when the orchestrator service is healthy, or timeout after 15 seconds +- [`agent_install`](#agent_install): Install the Puppet agent from a master +- [`backup_classification`](#backup_classification): A task to call the classification api and write to file +- [`cert_data`](#cert_data): Return certificate data related to the Puppet agent +- [`cert_valid_status`](#cert_valid_status): Check primary for valid state of a certificate +- [`code_manager`](#code_manager): Perform various code manager actions +- [`code_sync_status`](#code_sync_status): A task to confirm code is in sync accross the cluster for clusters with code manager configured +- [`divert_code_manager`](#divert_code_manager): Divert the code manager live-dir setting +- [`download`](#download): Download a file using curl +- [`enable_replica`](#enable_replica): Execute the enable replica puppet command +- [`filesize`](#filesize): Return the size of a file in bytes +- [`get_peadm_config`](#get_peadm_config): Run on a PE primary node to return the currently configured PEAdm parameters +- [`get_psql_version`](#get_psql_version): Run on a PE PSQL node to return the major version of the PSQL server currently installed +- [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output +- [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location +- [`mv`](#mv): Wrapper task for mv command +- [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string +- [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball +- [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console +- [`pe_uninstall`](#pe_uninstall): Uninstall Puppet Enterprise +- [`precheck`](#precheck): Return pre-check information about a system +- [`provision_replica`](#provision_replica): Execute the replica provision puppet command +- [`puppet_infra_upgrade`](#puppet_infra_upgrade): Execute the puppet infra upgrade command +- [`puppet_runonce`](#puppet_runonce): Run the Puppet agent one time +- [`rbac_token`](#rbac_token): Get and save an rbac token for the root user, admin rbac user +- [`read_file`](#read_file): Read the contents of a file +- [`reinstall_pe`](#reinstall_pe): Reinstall PE, only to be used to restore PE +- [`restore_classification`](#restore_classification): A short description of this task +- [`sign_csr`](#sign_csr): Submit a certificate signing request +- [`ssl_clean`](#ssl_clean): Clean an agent's certificate +- [`submit_csr`](#submit_csr): Submit a certificate signing request +- [`transform_classification_groups`](#transform_classification_groups): Transform the user groups from a source backup to a list of groups on the target server +- [`wait_until_service_ready`](#wait_until_service_ready): Return when the orchestrator service is healthy, or timeout after 15 seconds ### Plans #### Public Plans -* [`peadm::add_compiler`](#peadm--add_compiler): Add a new compiler to a PE architecture or replace an existing one with new configuration. -* [`peadm::add_database`](#peadm--add_database) -* [`peadm::add_replica`](#peadm--add_replica): Add or replace a replica host. -Supported use cases: -1: Adding a replica to an existing primary. -2: The existing replica is broken, we have a fresh new VM we want to provision the replica to. -* [`peadm::backup`](#peadm--backup): Backup puppet primary configuration -* [`peadm::backup_ca`](#peadm--backup_ca) -* [`peadm::convert`](#peadm--convert): Convert an existing PE cluster to a PEAdm-managed cluster -* [`peadm::install`](#peadm--install): Install a new PE cluster -* [`peadm::modify_certificate`](#peadm--modify_certificate): Modify the certificate of one or more targets -* [`peadm::restore`](#peadm--restore): Restore puppet primary configuration -* [`peadm::restore_ca`](#peadm--restore_ca) -* [`peadm::status`](#peadm--status): Return status information from one or more PE clusters in a table format -* [`peadm::upgrade`](#peadm--upgrade): Upgrade a PEAdm-managed cluster -* [`peadm::util::init_db_server`](#peadm--util--init_db_server) +- [`peadm::add_compiler`](#peadm--add_compiler): Add a new compiler to a PE architecture or replace an existing one with new configuration. +- [`peadm::add_database`](#peadm--add_database) +- [`peadm::add_replica`](#peadm--add_replica): Add or replace a replica host. + Supported use cases: + 1: Adding a replica to an existing primary. + 2: The existing replica is broken, we have a fresh new VM we want to provision the replica to. +- [`peadm::backup`](#peadm--backup): Backup puppet primary configuration +- [`peadm::backup_ca`](#peadm--backup_ca) +- [`peadm::convert`](#peadm--convert): Convert an existing PE cluster to a PEAdm-managed cluster +- [`peadm::install`](#peadm--install): Install a new PE cluster +- [`peadm::modify_certificate`](#peadm--modify_certificate): Modify the certificate of one or more targets +- [`peadm::restore`](#peadm--restore): Restore puppet primary configuration +- [`peadm::restore_ca`](#peadm--restore_ca) +- [`peadm::status`](#peadm--status): Return status information from one or more PE clusters in a table format +- [`peadm::upgrade`](#peadm--upgrade): Upgrade a PEAdm-managed cluster +- [`peadm::util::init_db_server`](#peadm--util--init_db_server) #### Private Plans -* `peadm::misc::divert_code_manager`: This plan exists to account for a scenario where a PE XL -* `peadm::modify_cert_extensions` -* `peadm::subplans::component_install`: Install a new PEADM component -* `peadm::subplans::configure`: Configure first-time classification and DR setup -* `peadm::subplans::db_populate`: Destructively (re)populates a new or existing database with the contents or a known good source -* `peadm::subplans::install`: Perform initial installation of Puppet Enterprise Extra Large -* `peadm::subplans::modify_certificate` -* `peadm::subplans::prepare_agent` -* `peadm::uninstall`: Single-entry-point plan for uninstalling Puppet Enterprise -* `peadm::util::code_sync_status` -* `peadm::util::copy_file` -* `peadm::util::db_disable_pglogical` -* `peadm::util::db_purge` -* `peadm::util::insert_csr_extension_requests` -* `peadm::util::retrieve_and_upload` -* `peadm::util::sanitize_pg_pe_conf` -* `peadm::util::update_classification`: Configure classification -* `peadm::util::update_db_setting`: Make updates to PuppetDB database settings +- `peadm::add_compiler`: Add a new compiler to a PE architecture or replace an existing one with new configuration. +- `peadm::add_replica`: Replace a replica host for a Standard or Large architecture. + Supported use cases: + 1: The existing replica is broken, we have a fresh new VM we want to provision the replica to. +- `peadm::convert_compiler_to_legacy` +- `peadm::misc::divert_code_manager`: This plan exists to account for a scenario where a PE XL +- `peadm::modify_cert_extensions` +- `peadm::subplans::component_install`: Install a new PEADM component +- `peadm::subplans::configure`: Configure first-time classification and DR setup +- `peadm::subplans::db_populate`: Destructively (re)populates a new or existing database with the contents or a known good source +- `peadm::subplans::install`: Perform initial installation of Puppet Enterprise Extra Large +- `peadm::subplans::modify_certificate` +- `peadm::subplans::prepare_agent` +- `peadm::uninstall`: Single-entry-point plan for uninstalling Puppet Enterprise +- `peadm::update_compiler_extensions` +- `peadm::util::code_sync_status` +- `peadm::util::copy_file` +- `peadm::util::db_disable_pglogical` +- `peadm::util::db_purge` +- `peadm::util::insert_csr_extension_requests` +- `peadm::util::retrieve_and_upload` +- `peadm::util::sanitize_pg_pe_conf` +- `peadm::util::update_classification`: Configure classification +- `peadm::util::update_db_setting`: Make updates to PuppetDB database settings ## Functions @@ -144,32 +151,22 @@ Returns: `Hash` Data type: `TargetSpec` - - ##### `replica_host` Data type: `Variant[TargetSpec, Undef]` - - ##### `primary_postgresql_host` Data type: `Variant[TargetSpec, Undef]` - - ##### `replica_postgresql_host` Data type: `Variant[TargetSpec, Undef]` - - ##### `compiler_hosts` Data type: `Variant[TargetSpec, Undef]` - - ### `peadm::assert_supported_bolt_version` Type: Puppet Language @@ -208,14 +205,10 @@ version number to check Data type: `String` - - ##### `permit_unsafe_versions` Data type: `Boolean` - - ### `peadm::bolt_version` Type: Ruby 4.x API @@ -241,12 +234,14 @@ is its certname. For strings, the certname is equal to the string. Undef input returns undef. #### `peadm::certname(Variant[Target, + String, Undef, Array[Target,1,1], Array[String,1,1], Array[Undef,1,1], - Array[Any,0,0]] $target)` + +Array[Any,0,0]] $target)` This function accepts a variety of data types which could represent single targets, and returns the certname corresponding to the input. @@ -272,8 +267,6 @@ Variant[Target, Array[Any,0,0]] ``` - - ### `peadm::check_version_and_known_hosts` Type: Puppet Language @@ -310,20 +303,14 @@ The r10k_known_hosts parameter Data type: `String` - - ##### `target_version` Data type: `String` - - ##### `r10k_known_hosts` Data type: `Optional[Peadm::Known_hosts]` - - ### `peadm::convert_hash` Type: Puppet Language @@ -605,20 +592,14 @@ Returns: `Any` Data type: `TargetSpec` - - ##### `transport` Data type: `String` - - ##### `message` Data type: `String` - - ### `peadm::file_content_upload` Type: Ruby 4.x API @@ -635,20 +616,14 @@ Returns: `Any` Data type: `String[1]` - - ##### `destination` Data type: `String[1]` - - ##### `*targets` Data type: `TargetOrTargets` - - ### `peadm::file_or_content` Type: Puppet Language @@ -665,20 +640,14 @@ Returns: `Any` Data type: `String` - - ##### `file` Data type: `Variant[String, Undef]` - - ##### `content` Data type: `Variant[String, Undef]` - - ### `peadm::flatten_compact` Type: Puppet Language @@ -695,8 +664,6 @@ Returns: `Any` Data type: `Array` - - ### `peadm::generate_pe_conf` Type: Puppet Language @@ -732,23 +699,23 @@ Returns: `Any` Data type: `Target` - - ### `peadm::get_targets` Type: Puppet Language Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - - It returns an Array[Target, 1, 0], rather than a Target - - It will accept undef and return [ ]. + +- It returns an Array[Target, 1, 0], rather than a Target +- It will accept undef and return [ ]. #### `peadm::get_targets(Variant[TargetSpec, Undef] $spec, Optional[Integer[1,1]] $count = undef)` Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - - It returns an Array[Target, 1, 0], rather than a Target - - It will accept undef and return [ ]. + +- It returns an Array[Target, 1, 0], rather than a Target +- It will accept undef and return [ ]. Returns: `Any` @@ -756,14 +723,10 @@ Returns: `Any` Data type: `Variant[TargetSpec, Undef]` - - ##### `count` Data type: `Optional[Integer[1,1]]` - - ### `peadm::migration_opts_default` Type: Puppet Language @@ -804,8 +767,6 @@ Returns: `Any` Data type: `String` - - ### `peadm::plan_step` Type: Ruby 4.x API @@ -822,14 +783,10 @@ Returns: `Any` Data type: `String` - - ##### `&block` Data type: `Callable` - - ### `peadm::recovery_opts_all` Type: Puppet Language @@ -896,14 +853,10 @@ Returns: `Any` Data type: `String` - - ##### `target` Data type: `TargetSpec` - - ## Data types ### `Peadm::Known_hosts` @@ -991,9 +944,9 @@ Struct[{ ### `Peadm::SingleTargetSpec` A SingleTargetSpec represents any String, Target or single-element array of -one or the other that can be passed to get_targets() to return an +one or the other that can be passed to get*targets() to return an Array[Target, 1, 1]. This is a constrained type variant of -Boltlib::TargetSpec for use when a _single_ target is valid, but multiple +Boltlib::TargetSpec for use when a \_single* target is valid, but multiple targets are not. Alias of `Variant[Pattern[/\A[^[:space:],]+\z/], Target, Array[Peadm::SingleTargetSpec, 1, 1]]` @@ -1564,17 +1517,17 @@ Add a new compiler to a PE architecture or replace an existing one with new conf The following parameters are available in the `peadm::add_compiler` plan: -* [`avail_group_letter`](#-peadm--add_compiler--avail_group_letter) -* [`compiler_host`](#-peadm--add_compiler--compiler_host) -* [`dns_alt_names`](#-peadm--add_compiler--dns_alt_names) -* [`primary_host`](#-peadm--add_compiler--primary_host) -* [`primary_postgresql_host`](#-peadm--add_compiler--primary_postgresql_host) +- [`avail_group_letter`](#-peadm--add_compiler--avail_group_letter) +- [`compiler_host`](#-peadm--add_compiler--compiler_host) +- [`dns_alt_names`](#-peadm--add_compiler--dns_alt_names) +- [`primary_host`](#-peadm--add_compiler--primary_host) +- [`primary_postgresql_host`](#-peadm--add_compiler--primary_postgresql_host) ##### `avail_group_letter` Data type: `Enum['A', 'B']` -_ Either A or B; whichever of the two letter designations the compiler is being assigned to +\_ Either A or B; whichever of the two letter designations the compiler is being assigned to Default value: `'A'` @@ -1582,13 +1535,13 @@ Default value: `'A'` Data type: `Peadm::SingleTargetSpec` -_ The hostname and certname of the new compiler +\_ The hostname and certname of the new compiler ##### `dns_alt_names` Data type: `Optional[String[1]]` -_ A comma_separated list of DNS alt names for the compiler +\_ A comma_separated list of DNS alt names for the compiler Default value: `undef` @@ -1596,13 +1549,13 @@ Default value: `undef` Data type: `Peadm::SingleTargetSpec` -_ The hostname and certname of the primary Puppet server +\_ The hostname and certname of the primary Puppet server ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` -_ The hostname and certname of the PE-PostgreSQL server with availability group $avail_group_letter +\_ The hostname and certname of the PE-PostgreSQL server with availability group $avail_group_letter Default value: `undef` @@ -1614,29 +1567,23 @@ The peadm::add_database class. The following parameters are available in the `peadm::add_database` plan: -* [`targets`](#-peadm--add_database--targets) -* [`primary_host`](#-peadm--add_database--primary_host) -* [`mode`](#-peadm--add_database--mode) -* [`begin_at_step`](#-peadm--add_database--begin_at_step) +- [`targets`](#-peadm--add_database--targets) +- [`primary_host`](#-peadm--add_database--primary_host) +- [`mode`](#-peadm--add_database--mode) +- [`begin_at_step`](#-peadm--add_database--begin_at_step) ##### `targets` Data type: `Peadm::SingleTargetSpec` - - ##### `primary_host` Data type: `Peadm::SingleTargetSpec` - - ##### `mode` Data type: `Optional[Enum['init', 'pair']]` - - Default value: `undef` ##### `begin_at_step` @@ -1653,8 +1600,6 @@ Optional[Enum[ 'finalize']] ``` - - Default value: `undef` ### `peadm::add_replica` @@ -1668,10 +1613,10 @@ Supported use cases: The following parameters are available in the `peadm::add_replica` plan: -* [`primary_host`](#-peadm--add_replica--primary_host) -* [`replica_host`](#-peadm--add_replica--replica_host) -* [`replica_postgresql_host`](#-peadm--add_replica--replica_postgresql_host) -* [`token_file`](#-peadm--add_replica--token_file) +- [`primary_host`](#-peadm--add_replica--primary_host) +- [`replica_host`](#-peadm--add_replica--replica_host) +- [`replica_postgresql_host`](#-peadm--add_replica--replica_postgresql_host) +- [`token_file`](#-peadm--add_replica--token_file) ##### `primary_host` @@ -1690,7 +1635,7 @@ Data type: `Peadm::SingleTargetSpec` Data type: `Optional[Peadm::SingleTargetSpec]` - The hostname and certname of the host with the replica PE-PosgreSQL database. -Can be a separate host in an XL architecture, or undef in Standard or Large. + Can be a separate host in an XL architecture, or undef in Standard or Large. Default value: `undef` @@ -1708,7 +1653,7 @@ Backup puppet primary configuration #### Examples -##### +##### ```puppet bolt plan run peadm::backup -t primary1.example.com @@ -1718,10 +1663,10 @@ bolt plan run peadm::backup -t primary1.example.com The following parameters are available in the `peadm::backup` plan: -* [`targets`](#-peadm--backup--targets) -* [`backup_type`](#-peadm--backup--backup_type) -* [`backup`](#-peadm--backup--backup) -* [`output_directory`](#-peadm--backup--output_directory) +- [`targets`](#-peadm--backup--targets) +- [`backup_type`](#-peadm--backup--backup_type) +- [`backup`](#-peadm--backup--backup) +- [`output_directory`](#-peadm--backup--output_directory) ##### `targets` @@ -1761,21 +1706,17 @@ The peadm::backup_ca class. The following parameters are available in the `peadm::backup_ca` plan: -* [`target`](#-peadm--backup_ca--target) -* [`output_directory`](#-peadm--backup_ca--output_directory) +- [`target`](#-peadm--backup_ca--target) +- [`output_directory`](#-peadm--backup_ca--output_directory) ##### `target` Data type: `Peadm::SingleTargetSpec` - - ##### `output_directory` Data type: `Optional[String]` - - Default value: `'/tmp'` ### `peadm::convert` @@ -1788,85 +1729,67 @@ management using PEAdm. The following parameters are available in the `peadm::convert` plan: -* [`primary_host`](#-peadm--convert--primary_host) -* [`replica_host`](#-peadm--convert--replica_host) -* [`compiler_hosts`](#-peadm--convert--compiler_hosts) -* [`primary_postgresql_host`](#-peadm--convert--primary_postgresql_host) -* [`replica_postgresql_host`](#-peadm--convert--replica_postgresql_host) -* [`compiler_pool_address`](#-peadm--convert--compiler_pool_address) -* [`internal_compiler_a_pool_address`](#-peadm--convert--internal_compiler_a_pool_address) -* [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) -* [`dns_alt_names`](#-peadm--convert--dns_alt_names) -* [`begin_at_step`](#-peadm--convert--begin_at_step) +- [`primary_host`](#-peadm--convert--primary_host) +- [`replica_host`](#-peadm--convert--replica_host) +- [`compiler_hosts`](#-peadm--convert--compiler_hosts) +- [`primary_postgresql_host`](#-peadm--convert--primary_postgresql_host) +- [`replica_postgresql_host`](#-peadm--convert--replica_postgresql_host) +- [`compiler_pool_address`](#-peadm--convert--compiler_pool_address) +- [`internal_compiler_a_pool_address`](#-peadm--convert--internal_compiler_a_pool_address) +- [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) +- [`dns_alt_names`](#-peadm--convert--dns_alt_names) +- [`begin_at_step`](#-peadm--convert--begin_at_step) ##### `primary_host` Data type: `Peadm::SingleTargetSpec` - - ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` - - Default value: `undef` ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `compiler_pool_address` Data type: `String` - - Default value: `$primary_host` ##### `internal_compiler_a_pool_address` Data type: `Optional[String]` - - Default value: `undef` ##### `internal_compiler_b_pool_address` Data type: `Optional[String]` - - Default value: `undef` ##### `dns_alt_names` Data type: `Array[String]` - - Default value: `[]` ##### `begin_at_step` @@ -1881,8 +1804,6 @@ Optional[Enum[ 'finalize']] ``` - - Default value: `undef` ### `peadm::install` @@ -1893,34 +1814,34 @@ Install a new PE cluster The following parameters are available in the `peadm::install` plan: -* [`compiler_pool_address`](#-peadm--install--compiler_pool_address) -* [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) -* [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) -* [`pe_installer_source`](#-peadm--install--pe_installer_source) -* [`ldap_config`](#-peadm--install--ldap_config) -* [`final_agent_state`](#-peadm--install--final_agent_state) -* [`stagingdir`](#-peadm--install--stagingdir) -* [`uploaddir`](#-peadm--install--uploaddir) -* [`primary_host`](#-peadm--install--primary_host) -* [`replica_host`](#-peadm--install--replica_host) -* [`compiler_hosts`](#-peadm--install--compiler_hosts) -* [`primary_postgresql_host`](#-peadm--install--primary_postgresql_host) -* [`replica_postgresql_host`](#-peadm--install--replica_postgresql_host) -* [`console_password`](#-peadm--install--console_password) -* [`version`](#-peadm--install--version) -* [`dns_alt_names`](#-peadm--install--dns_alt_names) -* [`pe_conf_data`](#-peadm--install--pe_conf_data) -* [`code_manager_auto_configure`](#-peadm--install--code_manager_auto_configure) -* [`r10k_remote`](#-peadm--install--r10k_remote) -* [`r10k_private_key_file`](#-peadm--install--r10k_private_key_file) -* [`r10k_private_key_content`](#-peadm--install--r10k_private_key_content) -* [`r10k_known_hosts`](#-peadm--install--r10k_known_hosts) -* [`deploy_environment`](#-peadm--install--deploy_environment) -* [`license_key_file`](#-peadm--install--license_key_file) -* [`license_key_content`](#-peadm--install--license_key_content) -* [`download_mode`](#-peadm--install--download_mode) -* [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) -* [`token_lifetime`](#-peadm--install--token_lifetime) +- [`compiler_pool_address`](#-peadm--install--compiler_pool_address) +- [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) +- [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) +- [`pe_installer_source`](#-peadm--install--pe_installer_source) +- [`ldap_config`](#-peadm--install--ldap_config) +- [`final_agent_state`](#-peadm--install--final_agent_state) +- [`stagingdir`](#-peadm--install--stagingdir) +- [`uploaddir`](#-peadm--install--uploaddir) +- [`primary_host`](#-peadm--install--primary_host) +- [`replica_host`](#-peadm--install--replica_host) +- [`compiler_hosts`](#-peadm--install--compiler_hosts) +- [`primary_postgresql_host`](#-peadm--install--primary_postgresql_host) +- [`replica_postgresql_host`](#-peadm--install--replica_postgresql_host) +- [`console_password`](#-peadm--install--console_password) +- [`version`](#-peadm--install--version) +- [`dns_alt_names`](#-peadm--install--dns_alt_names) +- [`pe_conf_data`](#-peadm--install--pe_conf_data) +- [`code_manager_auto_configure`](#-peadm--install--code_manager_auto_configure) +- [`r10k_remote`](#-peadm--install--r10k_remote) +- [`r10k_private_key_file`](#-peadm--install--r10k_private_key_file) +- [`r10k_private_key_content`](#-peadm--install--r10k_private_key_content) +- [`r10k_known_hosts`](#-peadm--install--r10k_known_hosts) +- [`deploy_environment`](#-peadm--install--deploy_environment) +- [`license_key_file`](#-peadm--install--license_key_file) +- [`license_key_content`](#-peadm--install--license_key_content) +- [`download_mode`](#-peadm--install--download_mode) +- [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) +- [`token_lifetime`](#-peadm--install--token_lifetime) ##### `compiler_pool_address` @@ -2004,156 +1925,116 @@ Default value: `undef` Data type: `Peadm::SingleTargetSpec` - - ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` - - Default value: `undef` ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `console_password` Data type: `String` - - ##### `version` Data type: `Peadm::Pe_version` - - Default value: `'2021.7.9'` ##### `dns_alt_names` Data type: `Optional[Array[String]]` - - Default value: `undef` ##### `pe_conf_data` Data type: `Optional[Hash]` - - Default value: `{}` ##### `code_manager_auto_configure` Data type: `Optional[Boolean]` - - Default value: `undef` ##### `r10k_remote` Data type: `Optional[String]` - - Default value: `undef` ##### `r10k_private_key_file` Data type: `Optional[String]` - - Default value: `undef` ##### `r10k_private_key_content` Data type: `Optional[Peadm::Pem]` - - Default value: `undef` ##### `r10k_known_hosts` Data type: `Optional[Peadm::Known_hosts]` - - Default value: `undef` ##### `deploy_environment` Data type: `Optional[String]` - - Default value: `undef` ##### `license_key_file` Data type: `Optional[String]` - - Default value: `undef` ##### `license_key_content` Data type: `Optional[String]` - - Default value: `undef` ##### `download_mode` Data type: `Enum['direct', 'bolthost']` - - Default value: `'bolthost'` ##### `permit_unsafe_versions` Data type: `Boolean` - - Default value: `false` ##### `token_lifetime` Data type: `String` - - Default value: `'1y'` ### `peadm::modify_certificate` @@ -2165,55 +2046,43 @@ setting DNS alternative names. The following parameters are available in the `peadm::modify_certificate` plan: -* [`targets`](#-peadm--modify_certificate--targets) -* [`primary_host`](#-peadm--modify_certificate--primary_host) -* [`add_extensions`](#-peadm--modify_certificate--add_extensions) -* [`remove_extensions`](#-peadm--modify_certificate--remove_extensions) -* [`dns_alt_names`](#-peadm--modify_certificate--dns_alt_names) -* [`force_regenerate`](#-peadm--modify_certificate--force_regenerate) +- [`targets`](#-peadm--modify_certificate--targets) +- [`primary_host`](#-peadm--modify_certificate--primary_host) +- [`add_extensions`](#-peadm--modify_certificate--add_extensions) +- [`remove_extensions`](#-peadm--modify_certificate--remove_extensions) +- [`dns_alt_names`](#-peadm--modify_certificate--dns_alt_names) +- [`force_regenerate`](#-peadm--modify_certificate--force_regenerate) ##### `targets` Data type: `TargetSpec` - - ##### `primary_host` Data type: `Peadm::SingleTargetSpec` - - ##### `add_extensions` Data type: `Hash` - - Default value: `{}` ##### `remove_extensions` Data type: `Array` - - Default value: `[]` ##### `dns_alt_names` Data type: `Optional[Array]` - - Default value: `undef` ##### `force_regenerate` Data type: `Boolean` - - Default value: `false` ### `peadm::restore` @@ -2222,7 +2091,7 @@ Restore puppet primary configuration #### Examples -##### +##### ```puppet bolt plan run peadm::restore -t primary1.example.com input_file=/tmp/peadm-backup.tar.gz @@ -2232,10 +2101,10 @@ bolt plan run peadm::restore -t primary1.example.com input_file=/tmp/peadm-backu The following parameters are available in the `peadm::restore` plan: -* [`targets`](#-peadm--restore--targets) -* [`restore_type`](#-peadm--restore--restore_type) -* [`restore`](#-peadm--restore--restore) -* [`input_file`](#-peadm--restore--input_file) +- [`targets`](#-peadm--restore--targets) +- [`restore_type`](#-peadm--restore--restore_type) +- [`restore`](#-peadm--restore--restore) +- [`input_file`](#-peadm--restore--input_file) ##### `targets` @@ -2273,28 +2142,22 @@ The peadm::restore_ca class. The following parameters are available in the `peadm::restore_ca` plan: -* [`target`](#-peadm--restore_ca--target) -* [`file_path`](#-peadm--restore_ca--file_path) -* [`recovery_directory`](#-peadm--restore_ca--recovery_directory) +- [`target`](#-peadm--restore_ca--target) +- [`file_path`](#-peadm--restore_ca--file_path) +- [`recovery_directory`](#-peadm--restore_ca--recovery_directory) ##### `target` Data type: `Peadm::SingleTargetSpec` - - ##### `file_path` Data type: `String` - - ##### `recovery_directory` Data type: `Optional[String]` - - Default value: `'/tmp/peadm_recovery'` ### `peadm::status` @@ -2313,11 +2176,11 @@ peadm::status($targets, 'table', true, true) The following parameters are available in the `peadm::status` plan: -* [`targets`](#-peadm--status--targets) -* [`format`](#-peadm--status--format) -* [`summarize`](#-peadm--status--summarize) -* [`verbose`](#-peadm--status--verbose) -* [`colors`](#-peadm--status--colors) +- [`targets`](#-peadm--status--targets) +- [`format`](#-peadm--status--format) +- [`summarize`](#-peadm--status--summarize) +- [`verbose`](#-peadm--status--verbose) +- [`colors`](#-peadm--status--colors) ##### `targets` @@ -2365,24 +2228,24 @@ Upgrade a PEAdm-managed cluster The following parameters are available in the `peadm::upgrade` plan: -* [`compiler_pool_address`](#-peadm--upgrade--compiler_pool_address) -* [`internal_compiler_a_pool_address`](#-peadm--upgrade--internal_compiler_a_pool_address) -* [`internal_compiler_b_pool_address`](#-peadm--upgrade--internal_compiler_b_pool_address) -* [`pe_installer_source`](#-peadm--upgrade--pe_installer_source) -* [`final_agent_state`](#-peadm--upgrade--final_agent_state) -* [`r10k_known_hosts`](#-peadm--upgrade--r10k_known_hosts) -* [`stagingdir`](#-peadm--upgrade--stagingdir) -* [`uploaddir`](#-peadm--upgrade--uploaddir) -* [`primary_host`](#-peadm--upgrade--primary_host) -* [`replica_host`](#-peadm--upgrade--replica_host) -* [`compiler_hosts`](#-peadm--upgrade--compiler_hosts) -* [`primary_postgresql_host`](#-peadm--upgrade--primary_postgresql_host) -* [`replica_postgresql_host`](#-peadm--upgrade--replica_postgresql_host) -* [`version`](#-peadm--upgrade--version) -* [`token_file`](#-peadm--upgrade--token_file) -* [`download_mode`](#-peadm--upgrade--download_mode) -* [`permit_unsafe_versions`](#-peadm--upgrade--permit_unsafe_versions) -* [`begin_at_step`](#-peadm--upgrade--begin_at_step) +- [`compiler_pool_address`](#-peadm--upgrade--compiler_pool_address) +- [`internal_compiler_a_pool_address`](#-peadm--upgrade--internal_compiler_a_pool_address) +- [`internal_compiler_b_pool_address`](#-peadm--upgrade--internal_compiler_b_pool_address) +- [`pe_installer_source`](#-peadm--upgrade--pe_installer_source) +- [`final_agent_state`](#-peadm--upgrade--final_agent_state) +- [`r10k_known_hosts`](#-peadm--upgrade--r10k_known_hosts) +- [`stagingdir`](#-peadm--upgrade--stagingdir) +- [`uploaddir`](#-peadm--upgrade--uploaddir) +- [`primary_host`](#-peadm--upgrade--primary_host) +- [`replica_host`](#-peadm--upgrade--replica_host) +- [`compiler_hosts`](#-peadm--upgrade--compiler_hosts) +- [`primary_postgresql_host`](#-peadm--upgrade--primary_postgresql_host) +- [`replica_postgresql_host`](#-peadm--upgrade--replica_postgresql_host) +- [`version`](#-peadm--upgrade--version) +- [`token_file`](#-peadm--upgrade--token_file) +- [`download_mode`](#-peadm--upgrade--download_mode) +- [`permit_unsafe_versions`](#-peadm--upgrade--permit_unsafe_versions) +- [`begin_at_step`](#-peadm--upgrade--begin_at_step) ##### `compiler_pool_address` @@ -2466,70 +2329,52 @@ Default value: `'/tmp'` Data type: `Peadm::SingleTargetSpec` - - ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` - - Default value: `undef` ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` - - Default value: `undef` ##### `version` Data type: `Optional[Peadm::Pe_version]` - - Default value: `undef` ##### `token_file` Data type: `Optional[String]` - - Default value: `undef` ##### `download_mode` Data type: `Enum[direct,bolthost]` - - Default value: `'bolthost'` ##### `permit_unsafe_versions` Data type: `Boolean` - - Default value: `false` ##### `begin_at_step` @@ -2546,8 +2391,6 @@ Optional[Enum[ 'finalize']] ``` - - Default value: `undef` ### `peadm::util::init_db_server` @@ -2558,38 +2401,29 @@ The peadm::util::init_db_server class. The following parameters are available in the `peadm::util::init_db_server` plan: -* [`db_host`](#-peadm--util--init_db_server--db_host) -* [`install_pe`](#-peadm--util--init_db_server--install_pe) -* [`pe_version`](#-peadm--util--init_db_server--pe_version) -* [`pe_platform`](#-peadm--util--init_db_server--pe_platform) +- [`db_host`](#-peadm--util--init_db_server--db_host) +- [`install_pe`](#-peadm--util--init_db_server--install_pe) +- [`pe_version`](#-peadm--util--init_db_server--pe_version) +- [`pe_platform`](#-peadm--util--init_db_server--pe_platform) ##### `db_host` Data type: `String[1]` - - ##### `install_pe` Data type: `Boolean` - - Default value: `false` ##### `pe_version` Data type: `String[1]` - - Default value: `'2023.5.0'` ##### `pe_platform` Data type: `String[1]` - - Default value: `'el-8-x86_64'` - diff --git a/documentation/convert.md b/documentation/convert.md index 6d2c8095..30242ef0 100644 --- a/documentation/convert.md +++ b/documentation/convert.md @@ -15,14 +15,14 @@ Prepare to run the plan against all servers in the PE infrastructure, using a pa "pe-xl-compiler-1.lab1.puppet.vm" ], - "compiler_pool_address": "puppet.lab1.puppet.vm", + "compiler_pool_address": "puppet.lab1.puppet.vm" } ``` -See the [install](install.md#reference-architectures) documentation for a list of supported architectures. Note that for convert, *all infrastructure being converted must already be functional*; you cannot use convert to add new systems to the infrastructure, nor can you use it to change your architecture. +See the [install](install.md#reference-architectures) documentation for a list of supported architectures. Note that for convert, _all infrastructure being converted must already be functional_; you cannot use convert to add new systems to the infrastructure, nor can you use it to change your architecture. ``` -bolt plan run peadm::convert --params @params.json +bolt plan run peadm::convert --params @params.json ``` ## Retry or resume plan @@ -30,3 +30,17 @@ bolt plan run peadm::convert --params @params.json This plan is broken down into steps. Normally, the plan runs through all the steps from start to finish. The name of each step is displayed during the plan run, as the step begins. The `begin_at_step` parameter can be used to facilitate re-running this plan after a failed attempt, skipping past any steps that already completed successfully on the first try and picking up again at the step specified. The step name to resume at can be read from the previous run logs. A full list of available values for this parameter can be viewed by running `bolt plan show peadm::convert`. + +## Convert compilers to legacy + +### Puppet Enterprise installed with puppetlabs-peadm version 3.21 or later + +To convert compilers to legacy compilers use the `peadm::convert_compiler_to_legacy` plan. This plan will create the needed Node group and Classifier rules to make the compilers legacy. Also will add certificate extensions to those nodes. + +```shell +bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com +``` + +### Puppet Enterprise installed with puppetlabs-peadm version prior to 3.21 + +Follow Steps 1 to 3 in the [Upgrade Puppet Enterprise with legacy compilers](upgrade_with_legacy_compilers.md) documentation. diff --git a/documentation/upgrade_with_legacy_compilers.md b/documentation/upgrade_with_legacy_compilers.md new file mode 100644 index 00000000..b33f4bf5 --- /dev/null +++ b/documentation/upgrade_with_legacy_compilers.md @@ -0,0 +1,45 @@ +# Upgrade Puppet Enterprise with legacy compilers + +## What is a legacy compiler and a current compiler + +As a legacy compiler we refer to a compiler that doesn't have PuppetDB. And a current Compiler is a compiler that has PuppetDB. By default, latest versions of Puppet enterprise comes with compilers that have PuppetDB.If your primary server and compilers are connected with high-latency links or congested network segments, you might experience better PuppetDB performance with legacy compilers. + +## Who is this documentation for + +For those users that have installed Puppet Enterprise with puppetlabs-peadm prior version 3.21 and manually converted their existing complilers (all of the or at least 1) to legacy compilers. + +## Who is this documentation not for + +For those users that have installed Puppet Enterprise with PEADM with 3.21 version or later, there is no need to follow this documentation. The install process will automatically have created the necessary configurations for you and you can use the `peadm::convert_compiler_to_legacy` plan if you need a legacy compiler. example: + +```shell +bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com +``` + +## How to upgrade Puppet Enterprise with legacy compilers + +### 1. Revert changes to the legacy compilers nodes + +Usually users pin the nodes in the Pe Master Node Group and then manually removing PuppetDB from compilers nodes. To revert this changes go to your Puppet Enterprise console and unpin the compilers nodes from the Group. + +### 2. Update certificate extensions for NON legacy compilers + +If you have NON legacy compilers in your infrastructure, you have to add a certificate extension to them that recognizes them as NON legacy compilers. To do this, execute the following plan: + +```shell +bolt plan run peadm::update_compiler_extensions primary_host=primary.example.com compiler_hosts=compiler1.example.com,compiler2.example.com +``` + +### 3. Use the convert legacy compiler plan + +Now that we have unpinned the compilers nodes from the PE Master node group, execute the following plan to convert your needed compilers to legacy compilers: + +```shell +bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com +``` + +The above will create the needed Node group and Classifier rules to make the compilers legacy. Also will add certificate extensions to those nodes. + +### 4. Upgrade Puppet Enterprise + +After you have completed the above steps, you can proceed with the upgrade of Puppet Enterprise as usual using the puppetlabs-peadm module. There is no need to do the above ever again. diff --git a/functions/oid.pp b/functions/oid.pp index 2fc735d0..0f03a43c 100644 --- a/functions/oid.pp +++ b/functions/oid.pp @@ -4,6 +4,7 @@ function peadm::oid ( case $short_name { 'peadm_role': { '1.3.6.1.4.1.34380.1.1.9812' } 'peadm_availability_group': { '1.3.6.1.4.1.34380.1.1.9813' } + 'peadm_legacy_compiler': { '1.3.6.1.4.1.34380.1.1.9814' } 'pp_application': { '1.3.6.1.4.1.34380.1.1.8' } 'pp_cluster': { '1.3.6.1.4.1.34380.1.1.16' } 'pp_role': { '1.3.6.1.4.1.34380.1.1.13' } diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp new file mode 100644 index 00000000..1311f2f2 --- /dev/null +++ b/manifests/setup/legacy_compiler_group.pp @@ -0,0 +1,50 @@ +# @api private +class peadm::setup::legacy_compiler_group ( + String[1] $primary_host +) { + Node_group { + purge_behavior => none, + } + + node_group { 'PE Legacy Compiler': + parent => 'PE Master', + rule => ['and', + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ], + classes => { + 'pe_repo' => {}, + 'puppet_enterprise::profile::master' => { 'code_manager_auto_configure' => true, 'replication_mode' => 'none' }, + }, + data => { + 'pe_repo' => { 'compile_master_pool_address' => $primary_host }, + }, + variables => { + 'pe_master' => true, + }, + } + + node_group { 'PE Legacy Compiler Group A': + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ], + } + + node_group { 'PE Legacy Compiler Group B': + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ], + } + + node_group { 'PE Compiler': + rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], + } +} diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 65c69044..f14d08bb 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -79,6 +79,12 @@ variables => { 'pe_master' => true }, } + # PE Compiler group comes from default PE and already has the pe compiler role + node_group { 'PE Compiler': + parent => 'PE Master', + rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], + } + # This group should pin the primary, and also map to any pe-postgresql nodes # which are part of the architecture. node_group { 'PE Database': @@ -115,6 +121,7 @@ rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false'], ], classes => { 'puppet_enterprise::profile::puppetdb' => { @@ -173,6 +180,7 @@ rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false'], ], classes => { 'puppet_enterprise::profile::puppetdb' => { @@ -192,4 +200,46 @@ }, }, } + + node_group { 'PE Legacy Compiler': + parent => 'PE Master', + rule => ['and', + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ], + classes => { + 'pe_repo' => {}, + 'puppet_enterprise::profile::master' => { 'code_manager_auto_configure' => true, 'replication_mode' => 'none' }, + }, + data => { + 'pe_repo' => { 'compile_master_pool_address' => $primary_host }, + }, + variables => { + 'pe_master' => true, + }, + } + + # Configure the A pool for legacy compilers. There are up to two pools for DR, each + # having an affinity for one "availability zone" or the other. + node_group { 'PE Legacy Compiler Group A': + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ], + } + + # Configure the B pool for legacy compilers. There are up to two pools for DR, each + # having an affinity for one "availability zone" or the other. + node_group { 'PE Legacy Compiler Group B': + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + ], + } } diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp new file mode 100644 index 00000000..69be62b3 --- /dev/null +++ b/plans/convert_compiler_to_legacy.pp @@ -0,0 +1,58 @@ +# @api private +plan peadm::convert_compiler_to_legacy ( + Peadm::SingleTargetSpec $primary_host, + TargetSpec $legacy_hosts, + Boolean $remove_pdb = false, +) { + $primary_target = peadm::get_targets($primary_host, 1) + $legacy_targets = peadm::get_targets($legacy_hosts) + + $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value + $error = getvar('cluster.error') + if $error { + fail_plan($error) + } + + $all_targets = peadm::flatten_compact([ + getvar('cluster.params.primary_host'), + getvar('cluster.params.replica_host'), + getvar('cluster.params.primary_postgresql_host'), + getvar('cluster.params.replica_postgresql_host'), + getvar('cluster.params.compiler_hosts'), + ]) + + if $remove_pdb { + run_command('puppet resource service puppet ensure=stopped', $legacy_targets) + run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_targets) + } + + apply($primary_target) { + class { 'peadm::setup::node_manager_yaml': + primary_host => $primary_target.peadm::certname(), + } + + class { 'peadm::setup::legacy_compiler_group': + primary_host => $primary_target.peadm::certname(), + } + } + + run_plan('peadm::update_compiler_extensions', compiler_hosts => $legacy_targets, primary_host => $primary_target, legacy => true) + + run_task('peadm::puppet_runonce', $legacy_targets) + run_task('peadm::puppet_runonce', $primary_target) + run_task('peadm::puppet_runonce', $all_targets) + + if $remove_pdb { + run_command('puppet resource package pe-puppetdb ensure=purged', $legacy_targets) + run_command('puppet resource user pe-puppetdb ensure=absent', $legacy_targets) + + run_command('rm -rf /etc/puppetlabs/puppetdb', $legacy_targets) + run_command('rm -rf /var/log/puppetlabs/puppetdb', $legacy_targets) + run_command('rm -rf /opt/puppetlabs/server/data/puppetdb', $legacy_targets) + } + + run_command('systemctl start pe-puppetserver.service', $legacy_targets) + run_command('puppet resource service puppet ensure=running', $legacy_targets) + + return("Converted host ${legacy_targets} to legacy compiler.") +} diff --git a/plans/subplans/component_install.pp b/plans/subplans/component_install.pp index df74079e..daf11688 100644 --- a/plans/subplans/component_install.pp +++ b/plans/subplans/component_install.pp @@ -22,6 +22,10 @@ peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => $avail_group_letter, } + } elsif $role == 'pe_compiler_legacy' { + $certificate_extensions = { + peadm::oid('peadm_role') => $role, + } } else { $certificate_extensions = { peadm::oid('peadm_role') => $role, diff --git a/plans/subplans/install.pp b/plans/subplans/install.pp index 7d48cb8a..88930218 100644 --- a/plans/subplans/install.pp +++ b/plans/subplans/install.pp @@ -278,6 +278,7 @@ extension_requests => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'A', + peadm::oid('peadm_legacy_compiler') => 'false', } ) }, @@ -286,6 +287,7 @@ extension_requests => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'B', + peadm::oid('peadm_legacy_compiler') => 'false', } ) }, diff --git a/plans/update_compiler_extensions.pp b/plans/update_compiler_extensions.pp new file mode 100644 index 00000000..784f919e --- /dev/null +++ b/plans/update_compiler_extensions.pp @@ -0,0 +1,25 @@ +# @api private +plan peadm::update_compiler_extensions ( + TargetSpec $compiler_hosts, + Peadm::SingleTargetSpec $primary_host, + Boolean $legacy = false, +) { + $primary_target = peadm::get_targets($primary_host, 1) + $host_targets = peadm::get_targets($compiler_hosts) + + run_plan('peadm::modify_certificate', $host_targets, + primary_host => $primary_target, + add_extensions => { peadm::oid('peadm_legacy_compiler') => String($legacy) }, + ) + + run_task('peadm::puppet_runonce', $primary_target) + run_task('peadm::puppet_runonce', $host_targets) + + if $legacy { + run_command('systemctl restart pe-puppetserver.service', $host_targets) + } else { + run_command('systemctl restart pe-puppetserver.service pe-puppetdb.service', $host_targets) + } + + return("Added legacy cert with value ${legacy} to compiler hosts ${compiler_hosts}") +} diff --git a/plans/upgrade.pp b/plans/upgrade.pp index b9adcd38..63375829 100644 --- a/plans/upgrade.pp +++ b/plans/upgrade.pp @@ -167,11 +167,25 @@ == $cert_extensions.dig($primary_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) } + $compiler_m1_nonlegacy_targets = $compiler_targets.filter |$target| { + ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_availability_group')) + == $cert_extensions.dig($primary_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) and + ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_legacy_compiler')) + == 'false') + } + $compiler_m2_targets = $compiler_targets.filter |$target| { ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_availability_group')) == $cert_extensions.dig($replica_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) } + $compiler_m2_nonlegacy_targets = $compiler_targets.filter |$target| { + ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_availability_group')) + == $cert_extensions.dig($replica_target[0].peadm::certname, peadm::oid('peadm_availability_group'))) and + ($cert_extensions.dig($target.peadm::certname, peadm::oid('peadm_legacy_compiler')) + == 'false') + } + peadm::plan_step('preparation') || { if $download_mode == 'bolthost' { # Download the PE tarball on the nodes that need it @@ -239,7 +253,7 @@ peadm::plan_step('upgrade-primary') || { # Shut down PuppetDB on CMs that use the PM's PDB PG. Use run_command instead # of run_task(service, ...) so that upgrading from 2018.1 works over PCP. - run_command('systemctl stop pe-puppetdb', $compiler_m1_targets) + run_command('systemctl stop pe-puppetdb', $compiler_m1_nonlegacy_targets) run_task('peadm::pe_install', $primary_postgresql_target, tarball => $upload_tarball_path, @@ -344,7 +358,7 @@ # Shut down PuppetDB on CMs that use the replica's PDB PG. Use run_command # instead of run_task(service, ...) so that upgrading from 2018.1 works # over PCP. - run_command('systemctl stop pe-puppetdb', $compiler_m2_targets) + run_command('systemctl stop pe-puppetdb', $compiler_m2_nonlegacy_targets) run_task('peadm::pe_install', $replica_postgresql_target, tarball => $upload_tarball_path, diff --git a/spec/acceptance/peadm_spec/plans/provision_test_cluster.pp b/spec/acceptance/peadm_spec/plans/provision_test_cluster.pp index aecd8197..4529dc9b 100644 --- a/spec/acceptance/peadm_spec/plans/provision_test_cluster.pp +++ b/spec/acceptance/peadm_spec/plans/provision_test_cluster.pp @@ -20,6 +20,9 @@ 'large': { ['primary', 'compiler'] } + 'large-with-two-compilers': { + ['primary', 'compiler', 'compiler'] + } 'large-with-dr': { ['primary', 'compiler', 'replica', 'compiler'] } From 80e07626c78184bb6e35ff498d966942e675997a Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Thu, 25 Jul 2024 10:44:52 +0100 Subject: [PATCH 02/37] Updating groups to cover ha scenarios --- manifests/setup/legacy_compiler_group.pp | 56 ++++++++++++++++------- manifests/setup/node_manager.pp | 58 ++++++++++++++++-------- 2 files changed, 79 insertions(+), 35 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 1311f2f2..5745eea9 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -7,41 +7,63 @@ } node_group { 'PE Legacy Compiler': - parent => 'PE Master', - rule => ['and', + parent => 'PE Master', + rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ], - classes => { - 'pe_repo' => {}, - 'puppet_enterprise::profile::master' => { 'code_manager_auto_configure' => true, 'replication_mode' => 'none' }, - }, - data => { - 'pe_repo' => { 'compile_master_pool_address' => $primary_host }, - }, - variables => { - 'pe_master' => true, + classes => { + 'puppet_enterprise::profile::master' => { + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}'], + # lint:endignore + 'puppetdb_port' => [8081], + }, }, } node_group { 'PE Legacy Compiler Group A': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], + classes => { + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_port' => [8081], + }, + }, + data => { + # Workaround for GH-118 + 'puppet_enterprise::profile::master::puppetdb' => { + 'ha_enabled_replicas' => [], + }, + }, } node_group { 'PE Legacy Compiler Group B': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], + classes => { + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_port' => [8081], + }, + }, + data => { + # Workaround for GH-118 + 'puppet_enterprise::profile::master::puppetdb' => { + 'ha_enabled_replicas' => [], + }, + }, } node_group { 'PE Compiler': diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index f14d08bb..47acd960 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -202,44 +202,66 @@ } node_group { 'PE Legacy Compiler': - parent => 'PE Master', - rule => ['and', - ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], + parent => 'PE Master', + rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], + ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], - classes => { - 'pe_repo' => {}, - 'puppet_enterprise::profile::master' => { 'code_manager_auto_configure' => true, 'replication_mode' => 'none' }, - }, - data => { - 'pe_repo' => { 'compile_master_pool_address' => $primary_host }, - }, - variables => { - 'pe_master' => true, + classes => { + 'puppet_enterprise::profile::master' => { + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}'], + # lint:endignore + 'puppetdb_port' => [8081], + }, }, } # Configure the A pool for legacy compilers. There are up to two pools for DR, each # having an affinity for one "availability zone" or the other. node_group { 'PE Legacy Compiler Group A': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'A'], ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], + classes => { + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_port' => [8081], + }, + }, + data => { + # Workaround for GH-118 + 'puppet_enterprise::profile::master::puppetdb' => { + 'ha_enabled_replicas' => [], + }, + }, } # Configure the B pool for legacy compilers. There are up to two pools for DR, each # having an affinity for one "availability zone" or the other. node_group { 'PE Legacy Compiler Group B': - ensure => 'present', - parent => 'PE Legacy Compiler', - rule => ['and', + ensure => 'present', + parent => 'PE Legacy Compiler', + rule => ['and', ['=', ['trusted', 'extensions', 'pp_auth_role'], 'pe_compiler'], ['=', ['trusted', 'extensions', peadm::oid('peadm_availability_group')], 'B'], ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'true'], ], + classes => { + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_port' => [8081], + }, + }, + data => { + # Workaround for GH-118 + 'puppet_enterprise::profile::master::puppetdb' => { + 'ha_enabled_replicas' => [], + }, + }, } } From a0ffeaa5464b2c64494252fbbdb9dfd7700d2a70 Mon Sep 17 00:00:00 2001 From: Aaron Shannon Date: Tue, 30 Jul 2024 15:45:25 +0100 Subject: [PATCH 03/37] PE-38768 classify compilers task added (#467) --- REFERENCE.md | 15 +++++++++ manifests/setup/legacy_compiler_group.pp | 6 ++-- manifests/setup/node_manager.pp | 2 +- tasks/classify_compilers.json | 15 +++++++++ tasks/classify_compilers.rb | 43 ++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 tasks/classify_compilers.json create mode 100755 tasks/classify_compilers.rb diff --git a/REFERENCE.md b/REFERENCE.md index 8d7c910e..abcf3b00 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -56,6 +56,7 @@ - [`backup_classification`](#backup_classification): A task to call the classification api and write to file - [`cert_data`](#cert_data): Return certificate data related to the Puppet agent - [`cert_valid_status`](#cert_valid_status): Check primary for valid state of a certificate +- [`classify_compilers`](#classify_compilers): Classify compilers as legacy or non-legacy - [`code_manager`](#code_manager): Perform various code manager actions - [`code_sync_status`](#code_sync_status): A task to confirm code is in sync accross the cluster for clusters with code manager configured - [`divert_code_manager`](#divert_code_manager): Divert the code manager live-dir setting @@ -1007,6 +1008,20 @@ Data type: `String` The certifcate name to check validation of +### `classify_compilers` + +Classify compilers as legacy or non-legacy + +**Supports noop?** false + +#### Parameters + +##### `compiler_hosts` + +Data type: `Array[String]` + +List of FQDNs of compilers + ### `code_manager` Perform various code manager actions diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 5745eea9..24041e4a 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -32,7 +32,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$peadm::setup::legacy_compiler_group::internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -54,7 +54,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$peadm::setup::legacy_compiler_group::internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -69,4 +69,4 @@ node_group { 'PE Compiler': rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], } -} +} \ No newline at end of file diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 47acd960..b6d5f096 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -253,7 +253,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, diff --git a/tasks/classify_compilers.json b/tasks/classify_compilers.json new file mode 100644 index 00000000..cb85a1dd --- /dev/null +++ b/tasks/classify_compilers.json @@ -0,0 +1,15 @@ +{ + "description": "Classify compilers as legacy or non-legacy", + "parameters": { + "compiler_hosts": { + "type": "Array[String]", + "description": "List of FQDNs of compilers" + } + }, + "implementations": [ + { + "name": "classify_compilers.rb", + "requirements": ["shell"] + } + ] +} \ No newline at end of file diff --git a/tasks/classify_compilers.rb b/tasks/classify_compilers.rb new file mode 100755 index 00000000..754f6ad9 --- /dev/null +++ b/tasks/classify_compilers.rb @@ -0,0 +1,43 @@ +#!/usr/bin/env ruby + +require 'json' +require 'open3' + +def classify_compiler(services) + if services.any? { |service| service['type'] == 'puppetdb' } + :non_legacy + else + :legacy + end +end + +params = JSON.parse(STDIN.read) +compiler_hosts = params['compiler_hosts'] + +legacy_compilers = [] +non_legacy_compilers = [] + +compiler_hosts.each do |compiler| + cmd = "puppet infra status --host #{compiler} --format=json" + stdout, stderr, status = Open3.capture3(cmd) + + if status.success? + services = JSON.parse(stdout) + classification = classify_compiler(services) + + if classification == :legacy + legacy_compilers << compiler + else + non_legacy_compilers << compiler + end + else + STDERR.puts "Error running command for #{compiler}: #{stderr}" + end +end + +result = { + 'legacy_compilers' => legacy_compilers, + 'compilers' => non_legacy_compilers +} + +puts result.to_json From 590af0b2fdf867b2b99b0e6bd69b076488e592ca Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Tue, 30 Jul 2024 15:50:30 +0100 Subject: [PATCH 04/37] (PE-38767) Adding legacy compilers to get_peadm_config (#456) Added legacy compilers section, and updated compilers with legacy compilers oid Co-authored-by: Neil Anderson --- tasks/get_peadm_config.rb | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tasks/get_peadm_config.rb b/tasks/get_peadm_config.rb index 30d8ad21..071229ac 100755 --- a/tasks/get_peadm_config.rb +++ b/tasks/get_peadm_config.rb @@ -47,6 +47,7 @@ def config 'primary_postgresql_host' => postgresql[primary_letter], 'replica_postgresql_host' => postgresql[replica_letter], 'compilers' => compilers.map { |c| c['certname'] }, + 'legacy_compilers' => legacy_compilers.map { |c| c['certname'] }, 'compiler_pool_address' => groups.dig('PE Master', 'config_data', 'pe_repo', 'compile_master_pool_address'), 'internal_compiler_a_pool_address' => groups.dig('PE Compiler Group B', 'classes', 'puppet_enterprise::profile::master', 'puppetdb_host', 1), 'internal_compiler_b_pool_address' => groups.dig('PE Compiler Group A', 'classes', 'puppet_enterprise::profile::master', 'puppetdb_host', 1), @@ -63,7 +64,11 @@ def config 'compilers' => { 'A' => compilers.select { |c| c['letter'] == 'A' }.map { |c| c['certname'] }, 'B' => compilers.select { |c| c['letter'] == 'B' }.map { |c| c['certname'] }, - } + }, + 'legacy_compilers' => { + 'A' => legacy_compilers.select { |c| c['letter'] == 'A' }.map { |c| c['certname'] }, + 'B' => legacy_compilers.select { |c| c['letter'] == 'B' }.map { |c| c['certname'] }, + }, }, } end @@ -81,7 +86,24 @@ def groups # Returns a list of compiler certnames and letters, based on a PuppetDB query def compilers @compilers ||= - pdb_query('inventory[certname,trusted.extensions] { trusted.extensions.pp_auth_role = "pe_compiler" }').map do |c| + pdb_query('inventory[certname,trusted.extensions] { + trusted.extensions.pp_auth_role = "pe_compiler" and + trusted.extensions."1.3.6.1.4.1.34380.1.1.9814" = "false" + }').map do |c| + { + 'certname' => c['certname'], + 'letter' => c.dig('trusted.extensions', '1.3.6.1.4.1.34380.1.1.9813'), + } + end + end + + # Returns a list of legacy compiler certnames and letters, based on a PuppetDB query + def legacy_compilers + @legacy_compilers ||= + pdb_query('inventory[certname,trusted.extensions] { + trusted.extensions.pp_auth_role = "pe_compiler" and + trusted.extensions."1.3.6.1.4.1.34380.1.1.9814" = "true" + }').map do |c| { 'certname' => c['certname'], 'letter' => c.dig('trusted.extensions', '1.3.6.1.4.1.34380.1.1.9813'), From e97569bb4741888445af6ba9d39deca0907b280d Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis <32846251+CoMfUcIoS@users.noreply.github.com> Date: Wed, 14 Aug 2024 09:48:28 +0100 Subject: [PATCH 05/37] (PE-38770) Install Plan accepts legacy_compilers key (#474) * feat(peadm): add support for legacy compilers - Added `legacy_compilers` parameter to `peadm::install`, `peadm::subplans::install`, and `peadm::subplans::configure` plans. - Updated `peadm::assert_supported_architecture` function to handle `legacy_compilers`. - Modified various plans to include `legacy_compilers` in the installation and configuration processes. - Updated documentation to reflect the changes and added examples for `legacy_compilers`. This change allows the PEADM module to support legacy compilers in addition to the standard compilers. * feat(assert_supported_architecture): normalize and combine compiler variables - Normalize `$legacy_compilers` and `$compiler_hosts` to arrays, handling both strings and arrays. - Combine the normalized arrays into `$all_compilers`. - Set `$has_compilers` to `undef` if `$all_compilers` is empty, otherwise set it to `true`. - Update tests to cover scenarios with both `compiler_hosts` and `legacy_compilers`. This change ensures that the function correctly handles different types for compiler variables and improves robustness. * Updated REFERENCE.md to document the new parameter. * style(tests): use single quotes for consistency in spec files Updated all double quotes to single quotes in the `assert_supported_architecture_spec.rb` and `install_spec.rb` files to maintain consistency in string literals across the test suite. * refactor(plans): rename legacy_targets to legacy_compiler_targets Renamed the variable `legacy_targets` to `legacy_compiler_targets` across multiple plans for better clarity and consistency. This change affects the following plans: - convert_compiler_to_legacy.pp - subplans/configure.pp - subplans/install.pp The new variable name more accurately reflects its purpose, which is to target legacy compiler hosts. * docs(install): correct grammar in installation requirements section Corrected the grammar in the installation requirements section by changing "Puppets must not be installed" to "Puppet must not be installed" for clarity and accuracy. --- REFERENCE.md | 53 ++++++- documentation/install.md | 130 +++++++++--------- functions/assert_supported_architecture.pp | 30 +++- plans/convert_compiler_to_legacy.pp | 26 ++-- plans/install.pp | 3 + plans/subplans/configure.pp | 6 + plans/subplans/install.pp | 31 ++++- .../assert_supported_architecture_spec.rb | 53 +++++++ spec/plans/subplans/install_spec.rb | 44 ++++-- 9 files changed, 281 insertions(+), 95 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index abcf3b00..2a9caebd 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -142,7 +142,7 @@ Type: Puppet Language Assert that the architecture given is a supported one -#### `peadm::assert_supported_architecture(TargetSpec $primary_host, Variant[TargetSpec, Undef] $replica_host = undef, Variant[TargetSpec, Undef] $primary_postgresql_host = undef, Variant[TargetSpec, Undef] $replica_postgresql_host = undef, Variant[TargetSpec, Undef] $compiler_hosts = undef)` +#### `peadm::assert_supported_architecture(TargetSpec $primary_host, Variant[TargetSpec, Undef] $replica_host = undef, Variant[TargetSpec, Undef] $primary_postgresql_host = undef, Variant[TargetSpec, Undef] $replica_postgresql_host = undef, Variant[TargetSpec, Undef] $compiler_hosts = undef, Variant[TargetSpec, Undef] $legacy_compilers = undef)` The peadm::assert_supported_architecture function. @@ -168,6 +168,17 @@ Data type: `Variant[TargetSpec, Undef]` Data type: `Variant[TargetSpec, Undef]` +<<<<<<< HEAD +======= + + +##### `legacy_compilers` + +Data type: `Variant[TargetSpec, Undef]` + + + +>>>>>>> d6467f9 ((PE-38770) Install Plan accepts legacy_compilers key (#474)) ### `peadm::assert_supported_bolt_version` Type: Puppet Language @@ -1829,6 +1840,7 @@ Install a new PE cluster The following parameters are available in the `peadm::install` plan: +<<<<<<< HEAD - [`compiler_pool_address`](#-peadm--install--compiler_pool_address) - [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) - [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) @@ -1857,6 +1869,37 @@ The following parameters are available in the `peadm::install` plan: - [`download_mode`](#-peadm--install--download_mode) - [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) - [`token_lifetime`](#-peadm--install--token_lifetime) +======= +* [`compiler_pool_address`](#-peadm--install--compiler_pool_address) +* [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) +* [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) +* [`pe_installer_source`](#-peadm--install--pe_installer_source) +* [`ldap_config`](#-peadm--install--ldap_config) +* [`final_agent_state`](#-peadm--install--final_agent_state) +* [`stagingdir`](#-peadm--install--stagingdir) +* [`uploaddir`](#-peadm--install--uploaddir) +* [`primary_host`](#-peadm--install--primary_host) +* [`replica_host`](#-peadm--install--replica_host) +* [`compiler_hosts`](#-peadm--install--compiler_hosts) +* [`legacy_compilers`](#-peadm--install--legacy_compilers) +* [`primary_postgresql_host`](#-peadm--install--primary_postgresql_host) +* [`replica_postgresql_host`](#-peadm--install--replica_postgresql_host) +* [`console_password`](#-peadm--install--console_password) +* [`version`](#-peadm--install--version) +* [`dns_alt_names`](#-peadm--install--dns_alt_names) +* [`pe_conf_data`](#-peadm--install--pe_conf_data) +* [`code_manager_auto_configure`](#-peadm--install--code_manager_auto_configure) +* [`r10k_remote`](#-peadm--install--r10k_remote) +* [`r10k_private_key_file`](#-peadm--install--r10k_private_key_file) +* [`r10k_private_key_content`](#-peadm--install--r10k_private_key_content) +* [`r10k_known_hosts`](#-peadm--install--r10k_known_hosts) +* [`deploy_environment`](#-peadm--install--deploy_environment) +* [`license_key_file`](#-peadm--install--license_key_file) +* [`license_key_content`](#-peadm--install--license_key_content) +* [`download_mode`](#-peadm--install--download_mode) +* [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) +* [`token_lifetime`](#-peadm--install--token_lifetime) +>>>>>>> d6467f9 ((PE-38770) Install Plan accepts legacy_compilers key (#474)) ##### `compiler_pool_address` @@ -1950,6 +1993,14 @@ Default value: `undef` Data type: `Optional[TargetSpec]` +Default value: `undef` + +##### `legacy_compilers` + +Data type: `Optional[TargetSpec]` + + + Default value: `undef` ##### `primary_postgresql_host` diff --git a/documentation/install.md b/documentation/install.md index 22e9d70c..72db9afc 100644 --- a/documentation/install.md +++ b/documentation/install.md @@ -1,69 +1,68 @@ -# Install Puppet Enterprise using the peadm module +# Install Puppet Enterprise using the PEADM module -The peadm module can be used to install Puppet Enterprise on new infrastructure. Supported architectures include Standard, Large, and Extra Large. +The PEADM module can be used to install Puppet Enterprise on new infrastructure. Supported architectures include Standard, Large, and Extra Large. -The peadm install plan creates a base install. Once a base cluster is installed, you may need to continue and perform additional configuration and adjustments to reach your target state, depending on your use case. +The PEADM install plan creates a base install. Once a base cluster is installed, you may need to continue and perform additional configuration and adjustments to reach your target state, depending on your use case. ## Reference Architectures -When installing a new PE cluster using peadm, there are several different host parameters which can be specified. At a minimum, you must always specify the primary parameter. Depending on which architecture you are deploying, other host parameters may be needed as well. The following is a list of the architectures peadm can install and the required parameters. - -* Standard - - primary -* Standard with DR - - primary - - primary-replica -* Large - - primary - - compilers -* Large with DR - - primary - - primary-replica - - compilers -* Extra Large - - primary - - pdb-database - - compilers (optional) -* Extra Large with DR - - primary - - primary-replica - - pdb-database - - pdb-database-replica - - compilers (optional) - -Supplying a combination of host parameters which does not match one of the supported architectures above will result in an unsupported architecture error. +When installing a new PE cluster using PEADM, several different host parameters can be specified. At a minimum, you must always specify the primary parameter. Depending on which architecture you are deploying, other host parameters may be needed as well. The following is a list of the architectures PEADM can install and the required parameters. + +- Standard + - primary +- Standard with DR + - primary + - primary-replica +- Large + - primary + - compilers +- Large with DR + - primary + - primary-replica + - compilers +- Extra Large + - primary + - pdb-database + - compilers (optional) +- Extra Large with DR + - primary + - primary-replica + - pdb-database + - pdb-database-replica + - compilers (optional) + +Supplying a combination of host parameters that do not match one of the supported architectures above will result in an unsupported architecture error. ## Requirements -* Puppet must not be installed on any of the target PE cluster hosts prior to beginning install. +Puppet must not be installed on any of the target PE cluster hosts before beginning installation. ## Usage ### Bolt 3 usage + We will name the bolt project `large_ha_peadm` in this example but the project name can be anything. 1. Install Bolt on a jumphost. This can be the primary, or any other system. (via package) 2. Run `mkdir large_ha_peadm && cd large_ha_peadm && bolt project init large_ha_peadm --modules puppetlabs-peadm` -4. Create an inventory file with connection information. Example included below. -5. Create a parameters file. Example included below. -6. Run `bolt plan run peadm::install --params @params.json ` after the inventory and params files are created. - +3. Create an inventory file with connection information. Example included below. +4. Create a parameters file. Example included below. +5. Run `bolt plan run peadm::install --params @params.json ` after the inventory and params files are created. ### Bolt 2 usage -1. Install Bolt on a jumphost. This can be the primary, or any other system. -2. Download or git clone the peadm module and put it somewhere on the jumphost. e.g. ~/modules/peadm. -3. Download or git clone the module dependencies, and put them somewhere on the jumphost. e.g. ~/modules/stdlib, ~/modules/node\_manager, etc. -4. Create an inventory file with connection information. Example included below. -5. Create a parameters file. Example included below. -6. Run the peadm::install plan with the inputs created. Example: +1. Install Bolt on a jumphost. This can be the primary or any other system. +2. Download or git clone the PEADM module and put it somewhere on the jumphost. e.g. ~/modules/peadm. +3. Download or git clone the module dependencies, and put them somewhere on the jumphost. e.g. ~/modules/stdlib, ~/modules/node_manager, etc. +4. Create an inventory file with connection information. Example included below. +5. Create a parameters file. Example included below. +6. Run the peadm::install plan with the inputs created. Example: bolt plan run peadm::install \ --inventory inventory.yaml \ --modulepath ~/modules \ --params @params.json - Example inventory.yaml Bolt inventory file: ```yaml @@ -85,9 +84,13 @@ groups: uri: 10.234.6.45 - name: pe-xl-compiler-1.lab1.puppet.vm uri: 10.234.14.131 + - name: pe-xl-legacy-compiler-0.lab1.puppet.vm + uri: 10.234.6.46 + - name: pe-xl-legacy-compiler-1.lab1.puppet.vm + uri: 10.234.6.47 ``` -Example params.json Bolt parameters file (shown: Extra Large with DR): +example params.json bolt parameters file (shown: extra large with dr): ```json { @@ -99,9 +102,12 @@ Example params.json Bolt parameters file (shown: Extra Large with DR): "pe-xl-compiler-0.lab1.puppet.vm", "pe-xl-compiler-1.lab1.puppet.vm" ], - + "legacy_compilers": [ + "pe-xl-legacy-compiler-0.lab1.puppet.vm", + "pe-xl-legacy-compiler-1.lab1.puppet.vm" + ], "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ], + "dns_alt_names": ["puppet", "puppet.lab1.puppet.vm"], "compiler_pool_address": "puppet.lab1.puppet.vm", "version": "2021.7.0" } @@ -113,12 +119,12 @@ Example params.json Bolt parameters file (shown: Standard): { "primary_host": "pe-xl-core-0.lab1.puppet.vm", "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ], + "dns_alt_names": ["puppet", "puppet.lab1.puppet.vm"], "version": "2021.7.9" } ``` -Review the [peadm::install plan](../plans/install.pp) to learn about more advanced installation options. For example, it is possible to: supply an ssh private key and git clone URL for a control-repo as part of installation; supply the LDAP configuration data for PE; specify where the installer tarball is uploaded; and similar complete automation tie-ins. +Review the [peadm::install plan](../plans/install.pp) to learn about more advanced installation options. For example, it is possible to: supply an SSH private key and git clone URL for a control-repo as part of installation; supply the LDAP configuration data for PE; specify where the installer tarball is uploaded; and similar complete automation tie-ins. ## Offline usage @@ -126,28 +132,27 @@ The peadm::install plan downloads installation content from an online repository The default staging directory is `/tmp`. If a different staging dir is being used, it can be specified using the `stagingdir` parameter to the peadm::install plan. -The content needed is the PE installation tarball for the target version. The installation content should be in the staging dir, and should have its original name. E.g. `/tmp/puppet-enterprise-2021.4.0-el-7-x86_64.tar.gz`. +The content needed is the PE installation tarball for the target version. The installation content should be in the staging dir and should have its original name. E.g. `/tmp/puppet-enterprise-2021.4.0-el-7-x86_64.tar.gz`. Installation content can be downloaded from [https://puppet.com/try-puppet/puppet-enterprise/download/](https://puppet.com/try-puppet/puppet-enterprise/download/). -If you wish to prevent the bolt host from transferring the installer tarball to the targets you can place the installer tar files on the target hosts in the _upload directory_. This can save time over slow networks, but is best to just perform the bolt task on the target node (puppetserver). The default _upload directory_ is `/tmp`. If a different upload dir is being used, it can be specified using the `uploaddir` parameter to the peadm::install plan. With default parameters the tar file will need to exist in the directories for offline configuration: +If you wish to prevent the bolt host from transferring the installer tarball to the targets you can place the installer tar files on the target hosts in the _upload directory_. This can save time over slow networks, but is best to just perform the bolt task on the target node (puppetserver). The default _upload directory_ is `/tmp`. If a different upload dir is being used, it can be specified using the `uploaddir` parameter to the peadm::install plan. With default parameters, the tar file will need to exist in the directories for offline configuration: -* /tmp on the Bolt host -* /tmp on the primary -* /tmp on the primary PuppetDB PostgreSQL (if used) -* /tmp on the replica PuppetDB PostgreSQL (if used) +- /tmp on the Bolt host +- /tmp on the primary +- /tmp on the primary PuppetDB PostgreSQL (if used) +- /tmp on the replica PuppetDB PostgreSQL (if used) ## Online usage -The peadm::install plan can be configured to download installation content directly to hosts. To configure online installation, set the `download_mode` parameter of the `peadm::install` plan to `direct`. The direct mode is often more efficient when PE hosts have a route to the internet. +The peadm::install plan can be configured to download installation content directly to hosts. To configure the online installation, set the `download_mode` parameter of the `peadm::install` plan to `direct`. The direct mode is often more efficient when PE hosts have a route to the internet. ## Hostnames and Certificate Names The various host parameters given to the peadm::install or peadm::action::install plans will be set as Puppet certificate names. You must use the names here that you want the servers to be identified as by Puppet. While it is not required that target names match hostnames, it _is_ required that target names be resolvable. - -In the event that Bolt will reach servers by IP address or external DNS name rather than internal DNS name or desired certname, a Bolt inventory file should be used to specify URIs for each name. For example: +If Bolt will reach servers by IP address or external DNS name rather than internal DNS name or desired certname, a Bolt inventory file should be used to specify URIs for each name. For example: ```yaml --- @@ -158,7 +163,7 @@ targets: uri: 10.234.14.131 ``` -A parameters JSON file can then reference the target names, which will become the Puppet certificate names, and Bolt will still be able to reach the systems by using the IP addresses or other DNS name specified as the URIs in the inventory.yaml file. +A parameters JSON file can then reference the target names, which will become the Puppet certificate names, and Bolt will still be able to reach the systems by using the IP addresses or other DNS names specified as the URIs in the inventory.yaml file. ```json { @@ -166,7 +171,7 @@ A parameters JSON file can then reference the target names, which will become th "replica_host": "pe-xl-core-1.lab1.puppet.vm", "console_password": "puppetlabs", - "dns_alt_names": [ "puppet", "puppet.lab1.puppet.vm" ], + "dns_alt_names": ["puppet", "puppet.lab1.puppet.vm"], "compiler_pool_address": "puppet.lab1.puppet.vm", "version": "2021.7.0" } @@ -174,12 +179,11 @@ A parameters JSON file can then reference the target names, which will become th ## Implementation Reference -Provisioning can be broken down into two actions: [install](../plans/action/install.pp), and [configure](../plans/subplans/configure.pp). Installation currently requires ssh access to the un-installed nodes, but configure can be performed using the Orchestrator transport if installation has already been completed. - -Besides getting Puppet Enterprise installed, the key configuration supporting Large and Extra Large architectures is laid out in four classification groups. Links are provided below to a Markdown document that describes the groups, and also to the Puppet manifest that actually configures them: +Provisioning can be broken down into two actions: [install](../plans/action/install.pp), and [configure](../plans/subplans/configure.pp). The installation currently requires SSH access to the uninstalled nodes, but configuration can be performed using the Orchestrator transport if the installation has already been completed. -* [classification.md](classification.md) -* [peadm::setup::node\_manager class](../manifests/setup/node_manager.pp) +Besides getting Puppet Enterprise installed, the key configuration supporting Large and Extra-large architectures is laid out in four classification groups. Links are provided below to a Markdown document that describes the groups, and also to the Puppet manifest that configures them: -The reference implementation uses trusted facts to put nodes in the right groups. Because the important puppet\_enterprise::\* class parameters and data are specified in the console, it should also be safe to have a pe.conf present on both the primary, and the primary replica nodes. +- [classification.md](classification.md) +- [peadm::setup::node_manager class](../manifests/setup/node_manager.pp) +The reference implementation uses trusted facts to put nodes in the right groups. Because the important puppet_enterprise::\* class parameters and data are specified in the console, it should also be safe to have a pe.conf present on both the primary and the primary replica nodes. diff --git a/functions/assert_supported_architecture.pp b/functions/assert_supported_architecture.pp index 0c705ddd..25709e86 100644 --- a/functions/assert_supported_architecture.pp +++ b/functions/assert_supported_architecture.pp @@ -5,7 +5,31 @@ function peadm::assert_supported_architecture ( Variant[TargetSpec, Undef] $primary_postgresql_host = undef, Variant[TargetSpec, Undef] $replica_postgresql_host = undef, Variant[TargetSpec, Undef] $compiler_hosts = undef, + Variant[TargetSpec, Undef] $legacy_compilers = undef, ) >> Hash { + # Normalize $legacy_compilers to an array + $legacy_compilers_array = $legacy_compilers ? { + undef => [], + String => [$legacy_compilers], + Array => $legacy_compilers, + default => fail("Unexpected type for \$legacy_compilers: ${legacy_compilers}"), + } + + # Normalize $compiler_hosts to an array + $compiler_hosts_array = $compiler_hosts ? { + undef => [], + String => [$compiler_hosts], + Array => $compiler_hosts, + default => fail("Unexpected type for \$compiler_hosts: ${compiler_hosts}"), + } + $all_compilers = $legacy_compilers_array + $compiler_hosts_array + + # Set $has_compilers to undef if $all_compilers is empty, otherwise set it to true + $has_compilers = empty($all_compilers) ? { + true => undef, + default => true, + } + $result = case [ !!($primary_host), !!($replica_host), @@ -13,13 +37,13 @@ function peadm::assert_supported_architecture ( !!($replica_postgresql_host), ] { [true, false, false, false]: { # Standard or Large, no DR - ({ 'disaster-recovery' => false, 'architecture' => $compiler_hosts ? { + ({ 'disaster-recovery' => false, 'architecture' => $has_compilers ? { undef => 'standard', default => 'large', } }) } [true, true, false, false]: { # Standard or Large, DR - ({ 'disaster-recovery' => true, 'architecture' => $compiler_hosts ? { + ({ 'disaster-recovery' => true, 'architecture' => $has_compilers ? { undef => 'standard', default => 'large', } }) @@ -44,7 +68,7 @@ function peadm::assert_supported_architecture ( <% if $replica_postgresql_host { -%> - pdb-database-replica <% } -%> - <% if $compiler_hosts { -%> + <% if $has_compilers { -%> - compilers <% } -%> diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 69be62b3..4433cdc0 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -5,7 +5,7 @@ Boolean $remove_pdb = false, ) { $primary_target = peadm::get_targets($primary_host, 1) - $legacy_targets = peadm::get_targets($legacy_hosts) + $legacy_compiler_targets = peadm::get_targets($legacy_hosts) $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value $error = getvar('cluster.error') @@ -22,8 +22,8 @@ ]) if $remove_pdb { - run_command('puppet resource service puppet ensure=stopped', $legacy_targets) - run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_targets) + run_command('puppet resource service puppet ensure=stopped', $legacy_compiler_targets) + run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_compiler_targets) } apply($primary_target) { @@ -36,23 +36,23 @@ } } - run_plan('peadm::update_compiler_extensions', compiler_hosts => $legacy_targets, primary_host => $primary_target, legacy => true) + run_plan('peadm::update_compiler_extensions', compiler_hosts => $legacy_compiler_targets, primary_host => $primary_target, legacy => true) - run_task('peadm::puppet_runonce', $legacy_targets) + run_task('peadm::puppet_runonce', $legacy_compiler_targets) run_task('peadm::puppet_runonce', $primary_target) run_task('peadm::puppet_runonce', $all_targets) if $remove_pdb { - run_command('puppet resource package pe-puppetdb ensure=purged', $legacy_targets) - run_command('puppet resource user pe-puppetdb ensure=absent', $legacy_targets) + run_command('puppet resource package pe-puppetdb ensure=purged', $legacy_compiler_targets) + run_command('puppet resource user pe-puppetdb ensure=absent', $legacy_compiler_targets) - run_command('rm -rf /etc/puppetlabs/puppetdb', $legacy_targets) - run_command('rm -rf /var/log/puppetlabs/puppetdb', $legacy_targets) - run_command('rm -rf /opt/puppetlabs/server/data/puppetdb', $legacy_targets) + run_command('rm -rf /etc/puppetlabs/puppetdb', $legacy_compiler_targets) + run_command('rm -rf /var/log/puppetlabs/puppetdb', $legacy_compiler_targets) + run_command('rm -rf /opt/puppetlabs/server/data/puppetdb', $legacy_compiler_targets) } - run_command('systemctl start pe-puppetserver.service', $legacy_targets) - run_command('puppet resource service puppet ensure=running', $legacy_targets) + run_command('systemctl start pe-puppetserver.service', $legacy_compiler_targets) + run_command('puppet resource service puppet ensure=running', $legacy_compiler_targets) - return("Converted host ${legacy_targets} to legacy compiler.") + return("Converted host ${legacy_compiler_targets} to legacy compiler.") } diff --git a/plans/install.pp b/plans/install.pp index fb1052c5..58b512ca 100644 --- a/plans/install.pp +++ b/plans/install.pp @@ -38,6 +38,7 @@ # Large Optional[TargetSpec] $compiler_hosts = undef, + Optional[TargetSpec] $legacy_compilers = undef, # Extra Large Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, @@ -85,6 +86,7 @@ # Large compiler_hosts => $compiler_hosts, + legacy_compilers => $legacy_compilers, # Extra Large primary_postgresql_host => $primary_postgresql_host, @@ -123,6 +125,7 @@ # Large compiler_hosts => $compiler_hosts, + legacy_compilers => $legacy_compilers, # Extra Large primary_postgresql_host => $primary_postgresql_host, diff --git a/plans/subplans/configure.pp b/plans/subplans/configure.pp index bccc0503..2afa0084 100644 --- a/plans/subplans/configure.pp +++ b/plans/subplans/configure.pp @@ -27,6 +27,7 @@ # Large Optional[TargetSpec] $compiler_hosts = undef, + Optional[TargetSpec] $legacy_compilers = undef, # Extra Large Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, @@ -51,6 +52,7 @@ $replica_target = peadm::get_targets($replica_host, 1) $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) $compiler_targets = peadm::get_targets($compiler_hosts) + $legacy_compiler_targets = peadm::get_targets($legacy_compilers) $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) # Ensure input valid for a supported architecture @@ -60,6 +62,7 @@ $primary_postgresql_host, $replica_postgresql_host, $compiler_hosts, + $legacy_compilers, ) # Source list of files on Primary and synchronize to new Replica @@ -74,6 +77,7 @@ run_plan('peadm::util::copy_file', peadm::flatten_compact([ $replica_target, $compiler_targets, + $legacy_compiler_targets, ]), source_host => $primary_target, path => $common_content_source @@ -139,6 +143,7 @@ $primary_target, $primary_postgresql_target, $compiler_targets, + $legacy_compiler_targets, $replica_target, $replica_postgresql_target, ])) @@ -161,6 +166,7 @@ $primary_postgresql_target, $replica_postgresql_target, $compiler_targets, + $legacy_compiler_targets, ])) return("Configuration of Puppet Enterprise ${arch['architecture']} succeeded.") diff --git a/plans/subplans/install.pp b/plans/subplans/install.pp index 88930218..d349e23e 100644 --- a/plans/subplans/install.pp +++ b/plans/subplans/install.pp @@ -41,6 +41,7 @@ # Large Optional[TargetSpec] $compiler_hosts = undef, + Optional[TargetSpec] $legacy_compilers = undef, # Extra Large Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, @@ -79,6 +80,7 @@ $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) $compiler_targets = peadm::get_targets($compiler_hosts) + $legacy_compiler_targets = peadm::get_targets($legacy_compilers) # Ensure input valid for a supported architecture $arch = peadm::assert_supported_architecture( @@ -87,6 +89,7 @@ $primary_postgresql_host, $replica_postgresql_host, $compiler_hosts, + $legacy_compilers, ) $all_targets = peadm::flatten_compact([ @@ -95,6 +98,7 @@ $replica_target, $replica_postgresql_target, $compiler_targets, + $legacy_compiler_targets, ]) $primary_targets = peadm::flatten_compact([ @@ -115,6 +119,7 @@ $agent_installer_targets = peadm::flatten_compact([ $compiler_targets, + $legacy_compiler_targets, $replica_target, ]) @@ -122,10 +127,14 @@ if $arch['disaster-recovery'] { $compiler_a_targets = $compiler_targets.filter |$index,$target| { $index % 2 == 0 } $compiler_b_targets = $compiler_targets.filter |$index,$target| { $index % 2 != 0 } + $legacy_a_targets = $legacy_compiler_targets.filter |$index,$target| { $index % 2 == 0 } + $legacy_b_targets = $legacy_compiler_targets.filter |$index,$target| { $index % 2 != 0 } } else { $compiler_a_targets = $compiler_targets $compiler_b_targets = [] + $legacy_a_targets = $legacy_compiler_targets + $legacy_b_targets = [] } $dns_alt_names_csv = $dns_alt_names.reduce |$csv,$x| { "${csv},${x}" } @@ -143,7 +152,7 @@ true } elsif $replica_host { true - } elsif $compiler_hosts { + } elsif $compiler_hosts or $legacy_compilers { true } else { $code_manager_auto_configure @@ -179,7 +188,7 @@ # puppet and are present in PuppetDB, it is not necessary anymore. $puppetdb_database_temp_config = { 'puppet_enterprise::profile::database::puppetdb_hosts' => ( - $compiler_targets + $primary_target + $replica_target + $compiler_targets + $legacy_compiler_targets + $primary_target + $replica_target ).map |$t| { $t.peadm::certname() }, } @@ -291,6 +300,24 @@ } ) }, + background('compiler-a-csr.yaml') || { + run_plan('peadm::util::insert_csr_extension_requests', $legacy_a_targets, + extension_requests => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'A', + peadm::oid('peadm_legacy_compiler') => 'true', + } + ) + }, + background('compiler-b-csr.yaml') || { + run_plan('peadm::util::insert_csr_extension_requests', $legacy_b_targets, + extension_requests => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'B', + peadm::oid('peadm_legacy_compiler') => 'true', + } + ) + }, background('primary-postgresql-csr.yaml') || { run_plan('peadm::util::insert_csr_extension_requests', $primary_postgresql_target, extension_requests => { diff --git a/spec/functions/assert_supported_architecture_spec.rb b/spec/functions/assert_supported_architecture_spec.rb index d48342f3..5f67a33b 100644 --- a/spec/functions/assert_supported_architecture_spec.rb +++ b/spec/functions/assert_supported_architecture_spec.rb @@ -24,6 +24,15 @@ let(:compiler_hosts) do 'pup-c1.puppet.vm' end + let(:legacy_compilers) do + 'pup-c2.puppet.vm' + end + let(:compiler_hosts_array) do + ['pup-c1.puppet.vm'] + end + let(:legacy_compilers_array) do + ['pup-c2.puppet.vm'] + end it { is_expected.to run.with_params(primary_host) @@ -59,6 +68,28 @@ 'disaster-recovery' => false, 'architecture' => 'large') end + it do + is_expected.to run.with_params(primary_host, + nil, + nil, + nil, + compiler_hosts, + legacy_compilers) + .and_return('supported' => true, + 'disaster-recovery' => false, + 'architecture' => 'large') + end + it do + is_expected.to run.with_params(primary_host, + nil, + nil, + nil, + nil, + legacy_compilers) + .and_return('supported' => true, + 'disaster-recovery' => false, + 'architecture' => 'large') + end it do is_expected.to run.with_params(primary_host, @@ -81,4 +112,26 @@ 'disaster-recovery' => false, 'architecture' => 'extra-large') end + it do + is_expected.to run.with_params(primary_host, + nil, + primary_postgresql_host, + nil, + compiler_hosts, + legacy_compilers) + .and_return('supported' => true, + 'disaster-recovery' => false, + 'architecture' => 'extra-large') + end + it do + is_expected.to run.with_params(primary_host, + nil, + primary_postgresql_host, + nil, + compiler_hosts_array, + legacy_compilers_array) + .and_return('supported' => true, + 'disaster-recovery' => false, + 'architecture' => 'extra-large') + end end diff --git a/spec/plans/subplans/install_spec.rb b/spec/plans/subplans/install_spec.rb index f4ed169e..c83a47a7 100644 --- a/spec/plans/subplans/install_spec.rb +++ b/spec/plans/subplans/install_spec.rb @@ -12,7 +12,15 @@ allow_task('peadm::precheck').return_for_targets( 'primary' => { 'hostname' => 'primary', - 'platform' => 'el-7.11-x86_64' + 'platform' => 'el-7.11-x86_64', + }, + 'compiler1' => { + 'hostname' => 'compiler1', + 'platform' => 'el-7.11-x86_64', + }, + 'compiler2' => { + 'hostname' => 'compiler2', + 'platform' => 'el-7.11-x86_64', }, ) @@ -39,9 +47,9 @@ it 'minimum variables to run' do params = { - 'primary_host' => 'primary', + 'primary_host' => 'primary', 'console_password' => 'puppetLabs123!', - 'version' => '2019.8.12', + 'version' => '2019.8.12', } expect(run_plan('peadm::subplans::install', params)).to be_ok @@ -49,10 +57,10 @@ it 'installs 2023.4 without r10k_known_hosts' do params = { - 'primary_host' => 'primary', - 'console_password' => 'puppetLabs123!', - 'version' => '2023.4.0', - 'r10k_remote' => 'git@github.com:puppetlabs/nothing', + 'primary_host' => 'primary', + 'console_password' => 'puppetLabs123!', + 'version' => '2023.4.0', + 'r10k_remote' => 'git@github.com:puppetlabs/nothing', 'r10k_private_key_content' => '-----BEGINfoo', } @@ -61,16 +69,16 @@ it 'installs 2023.4+ with r10k_private_key and r10k_known_hosts' do params = { - 'primary_host' => 'primary', - 'console_password' => 'puppetLabs123!', - 'version' => '2023.4.0', - 'r10k_remote' => 'git@github.com:puppetlabs/nothing', + 'primary_host' => 'primary', + 'console_password' => 'puppetLabs123!', + 'version' => '2023.4.0', + 'r10k_remote' => 'git@github.com:puppetlabs/nothing', 'r10k_private_key_content' => '-----BEGINfoo', - 'r10k_known_hosts' => [ + 'r10k_known_hosts' => [ { 'name' => 'test', 'type' => 'key-type', - 'key' => 'abcdef', + 'key' => 'abcdef', }, ], 'permit_unsafe_versions' => true, @@ -78,4 +86,14 @@ expect(run_plan('peadm::subplans::install', params)).to be_ok end + + it 'installs 2021.7.9 with legacy compilers' do + params = { + 'primary_host' => 'primary', + 'console_password' => 'puppetLabs123!', + 'version' => '2021.7.9', + 'legacy_compilers' => ['compiler1', 'compiler2'], + } + expect(run_plan('peadm::subplans::install', params)).to be_ok + end end From 496e4bce439b04ed3afc8cce48e05ad9389bee28 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis <32846251+CoMfUcIoS@users.noreply.github.com> Date: Thu, 15 Aug 2024 16:21:06 +0100 Subject: [PATCH 06/37] (PE-38771) Convert plan accepts legacy compilers key in params.json (#476) * feat(convert): add support for legacy compilers - Introduced `legacy_compilers` parameter to handle legacy compiler hosts. - Added logic to filter and categorize legacy compiler targets. - Updated certificate modification steps to include legacy compiler targets with appropriate extensions. * feat(convert): ensure all targets are up-to-date post conversion - Added a step to run puppet on all targets after restarting services. - Ensures all targets are fully up-to-date after conversion. * docs(convert): update documentation for PEADM module - Corrected the module name from 'peadm' to 'PEADM' for consistency. - Added 'legacy_compilers' section in the example JSON parameters. - Improved clarity in the description of the `begin_at_step` parameter. - Fixed minor grammatical issues in the instructions for converting compilers to legacy. * - Updated REFERENCE.md to include the new parameter. --- REFERENCE.md | 22 ++++++++++++++++ documentation/convert.md | 13 ++++++---- plans/convert.pp | 55 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+), 5 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 2a9caebd..5a6c2b90 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -1755,6 +1755,7 @@ management using PEAdm. The following parameters are available in the `peadm::convert` plan: +<<<<<<< HEAD - [`primary_host`](#-peadm--convert--primary_host) - [`replica_host`](#-peadm--convert--replica_host) - [`compiler_hosts`](#-peadm--convert--compiler_hosts) @@ -1765,6 +1766,19 @@ The following parameters are available in the `peadm::convert` plan: - [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) - [`dns_alt_names`](#-peadm--convert--dns_alt_names) - [`begin_at_step`](#-peadm--convert--begin_at_step) +======= +* [`primary_host`](#-peadm--convert--primary_host) +* [`replica_host`](#-peadm--convert--replica_host) +* [`compiler_hosts`](#-peadm--convert--compiler_hosts) +* [`legacy_compilers`](#-peadm--convert--legacy_compilers) +* [`primary_postgresql_host`](#-peadm--convert--primary_postgresql_host) +* [`replica_postgresql_host`](#-peadm--convert--replica_postgresql_host) +* [`compiler_pool_address`](#-peadm--convert--compiler_pool_address) +* [`internal_compiler_a_pool_address`](#-peadm--convert--internal_compiler_a_pool_address) +* [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) +* [`dns_alt_names`](#-peadm--convert--dns_alt_names) +* [`begin_at_step`](#-peadm--convert--begin_at_step) +>>>>>>> 671839c ((PE-38771) Convert plan accepts legacy compilers key in params.json (#476)) ##### `primary_host` @@ -1780,6 +1794,14 @@ Default value: `undef` Data type: `Optional[TargetSpec]` +Default value: `undef` + +##### `legacy_compilers` + +Data type: `Optional[TargetSpec]` + + + Default value: `undef` ##### `primary_postgresql_host` diff --git a/documentation/convert.md b/documentation/convert.md index 30242ef0..c0ec001f 100644 --- a/documentation/convert.md +++ b/documentation/convert.md @@ -1,6 +1,6 @@ -# Convert infrastructure for use with the peadm module +# Convert infrastructure for use with the PEADM module -The peadm::convert plan can be used to adopt manually deployed infrastructure for use with peadm, or to adopt infrastructure deployed with an older version of peadm. +The peadm::convert plan can be used to adopt manually deployed infrastructure for use with PEADM or to adopt infrastructure deployed with an older version of peadm. ## Convert an Existing Deployment @@ -14,7 +14,10 @@ Prepare to run the plan against all servers in the PE infrastructure, using a pa "pe-xl-compiler-0.lab1.puppet.vm", "pe-xl-compiler-1.lab1.puppet.vm" ], - + "legacy_compilers": [ + "pe-xl-legacy-compiler-0.lab1.puppet.vm", + "pe-xl-legacy-compiler-1.lab1.puppet.vm" + ], "compiler_pool_address": "puppet.lab1.puppet.vm" } ``` @@ -29,13 +32,13 @@ bolt plan run peadm::convert --params @params.json This plan is broken down into steps. Normally, the plan runs through all the steps from start to finish. The name of each step is displayed during the plan run, as the step begins. -The `begin_at_step` parameter can be used to facilitate re-running this plan after a failed attempt, skipping past any steps that already completed successfully on the first try and picking up again at the step specified. The step name to resume at can be read from the previous run logs. A full list of available values for this parameter can be viewed by running `bolt plan show peadm::convert`. +The `begin_at_step` parameter can be used to facilitate re-running this plan after a failed attempt, skipping past any steps that were already completed successfully on the first try and picking up again at the step specified. The step name to resume can be read from the previous run logs. A full list of available values for this parameter can be viewed by running `bolt plan show peadm::convert`. ## Convert compilers to legacy ### Puppet Enterprise installed with puppetlabs-peadm version 3.21 or later -To convert compilers to legacy compilers use the `peadm::convert_compiler_to_legacy` plan. This plan will create the needed Node group and Classifier rules to make the compilers legacy. Also will add certificate extensions to those nodes. +To convert compilers to legacy compilers use the `peadm::convert_compiler_to_legacy` plan. This plan will create the needed Node group and Classifier rules to make compilers legacy. Also will add certificate extensions to those nodes. ```shell bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com diff --git a/plans/convert.pp b/plans/convert.pp index 1995a0b0..f452e81c 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -10,6 +10,7 @@ # Large Optional[TargetSpec] $compiler_hosts = undef, + Optional[TargetSpec] $legacy_compilers = undef, # Extra Large Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, @@ -36,6 +37,7 @@ $replica_target = peadm::get_targets($replica_host, 1) $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) $compiler_targets = peadm::get_targets($compiler_hosts) + $legacy_compiler_targets = peadm::get_targets($legacy_compilers) $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) $all_targets = peadm::flatten_compact([ @@ -43,6 +45,7 @@ $replica_target, $replica_postgresql_target, $compiler_targets, + $legacy_compiler_targets, $primary_postgresql_target, ]) @@ -53,6 +56,7 @@ $primary_postgresql_host, $replica_postgresql_host, $compiler_hosts, + $legacy_compilers, ) out::message('# Gathering information') @@ -115,10 +119,36 @@ $index % 2 != 0 } } + $legacy_compiler_a_targets = $legacy_compiler_targets.filter |$index,$target| { + $exts = $cert_extensions[$target.peadm::certname()] + if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { + $exts[peadm::oid('peadm_availability_group')] == 'A' + } + elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) { + $exts[peadm::oid('pp_cluster')] == 'A' + } + else { + $index % 2 == 0 + } + } + $legacy_compiler_b_targets = $legacy_compiler_targets.filter |$index,$target| { + $exts = $cert_extensions[$target.peadm::certname()] + if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { + $exts[peadm::oid('peadm_availability_group')] == 'B' + } + elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) { + $exts[peadm::oid('pp_cluster')] == 'B' + } + else { + $index % 2 != 0 + } + } } else { $compiler_a_targets = $compiler_targets $compiler_b_targets = [] + $legacy_compiler_a_targets = $legacy_compiler_targets + $legacy_compiler_b_targets = [] } # Modify csr_attributes.yaml and insert the peadm-specific OIDs to identify @@ -185,6 +215,7 @@ add_extensions => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'A', + peadm::oid('peadm_legacy_compiler') => 'false', }, ) }, @@ -194,6 +225,27 @@ add_extensions => { peadm::oid('pp_auth_role') => 'pe_compiler', peadm::oid('peadm_availability_group') => 'B', + peadm::oid('peadm_legacy_compiler') => 'false', + }, + ) + }, + background('modify-compilers-a-certs') || { + run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, + primary_host => $primary_target, + add_extensions => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'A', + peadm::oid('peadm_legacy_compiler') => 'true', + }, + ) + }, + background('modify-compilers-b-certs') || { + run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, + primary_host => $primary_target, + add_extensions => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'B', + peadm::oid('peadm_legacy_compiler') => 'true', }, ) }, @@ -252,6 +304,9 @@ # completion run_command('systemctl restart pe-puppetserver.service pe-puppetdb.service', $all_targets - $primary_target - $primary_postgresql_target - $replica_postgresql_target) + + # Run puppet on all targets again to ensure everything is fully up-to-date + run_task('peadm::puppet_runonce', $all_targets) } return("Conversion to peadm Puppet Enterprise ${arch['architecture']} completed.") From 9d584f291b4ebf8df2cb838f2cbde574e1d50038 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis <32846251+CoMfUcIoS@users.noreply.github.com> Date: Mon, 2 Sep 2024 10:26:19 +0100 Subject: [PATCH 07/37] feat(convert): add check for legacy compilers (#483) - Introduced a new task `check_legacy_compilers.rb` to verify legacy compilers. - Updated `convert.pp` to run the new task and display warnings if legacy compilers are detected. --- plans/convert.pp | 11 ++++++ tasks/check_legacy_compilers.rb | 67 +++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100755 tasks/check_legacy_compilers.rb diff --git a/plans/convert.pp b/plans/convert.pp index f452e81c..e697e87d 100644 --- a/plans/convert.pp +++ b/plans/convert.pp @@ -309,5 +309,16 @@ run_task('peadm::puppet_runonce', $all_targets) } + if $legacy_compilers { +# lint:ignore:strict_indent + $warning_msg = run_task('peadm::check_legacy_compilers', $primary_host, legacy_compilers => $legacy_compilers.join(',') ).first.message + if $warning_msg.length > 0 { + out::message(@("WARN"/L)) + WARNING: ${warning_msg} + | WARN + } +# lint:endignore + } + return("Conversion to peadm Puppet Enterprise ${arch['architecture']} completed.") } diff --git a/tasks/check_legacy_compilers.rb b/tasks/check_legacy_compilers.rb new file mode 100755 index 00000000..eadb4fbc --- /dev/null +++ b/tasks/check_legacy_compilers.rb @@ -0,0 +1,67 @@ +#!/opt/puppetlabs/puppet/bin/ruby +# frozen_string_literal: true + +require 'json' +require 'uri' +require 'net/http' +require 'puppet' + +# CheckLegacyCompilers task class +class CheckLegacyCompilers + def initialize(params) + @nodes = params['legacy_compilers'].split(',') if params['legacy_compilers'].is_a?(String) + end + + def execute! + pinned_nodes = [] + @nodes.each do |node| + node_classification = get_node_classification(node) + pinned = false + node_classification['groups'].each do |group| + if group['name'] == 'PE Master' + pinned_nodes << node + pinned = true + end + end + next if pinned + next unless node_classification.key?('parameters') + next unless node_classification['parameters'].key?('pe_master') + if node_classification['parameters']['pe_master'] + pinned_nodes << node + end + end + + return unless !pinned_nodes.empty? + puts 'The following legacy compilers are classified as Puppet primary:' + puts pinned_nodes.join(', ') + puts 'You will not be able to upgrade if you dont remediate this.' + end + + def https(port) + https = Net::HTTP.new('localhost', port) + https.use_ssl = true + https.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) + https.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) + https.verify_mode = OpenSSL::SSL::VERIFY_NONE + https + end + + def get_node_classification(certname) + pdb = https(4433) + pdb_request = Net::HTTP::Post.new('/classifier-api/v2/classified/nodes/' + certname) + pdb_request['Content-Type'] = 'application/json' + + response = JSON.parse(pdb.request(pdb_request).body) + + response + end +end + +# Run the task unless an environment flag has been set, signaling not to. The +# environment flag is used to disable auto-execution and enable Ruby unit +# testing of this task. +unless ENV['RSPEC_UNIT_TEST_MODE'] + Puppet.initialize_settings + task = CheckLegacyCompilers.new(JSON.parse(STDIN.read)) + task.execute! +end From 3a5f4fbbd33740da9e1dc9d2fa35b46f703ccd8b Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 2 Sep 2024 12:14:06 +0100 Subject: [PATCH 08/37] fix: update messaging for legacy compilers check - Clarified the message indicating which legacy compilers are classified as Puppet primary nodes. - Improved the remediation message to specify that the upgrade cannot proceed until the compilers are no longer classified as Puppet primary nodes. --- tasks/check_legacy_compilers.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/check_legacy_compilers.rb b/tasks/check_legacy_compilers.rb index eadb4fbc..65330ba9 100755 --- a/tasks/check_legacy_compilers.rb +++ b/tasks/check_legacy_compilers.rb @@ -32,9 +32,9 @@ def execute! end return unless !pinned_nodes.empty? - puts 'The following legacy compilers are classified as Puppet primary:' + puts 'The following legacy compilers are classified as Puppet primary nodes:' puts pinned_nodes.join(', ') - puts 'You will not be able to upgrade if you dont remediate this.' + puts 'You will not be able to upgrade until these compilers are no longer classified as Puppet primary nodes.' end def https(port) From 5a053855f90a7a78ed673fd7c21717c7e6fd6120 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 2 Sep 2024 12:26:46 +0100 Subject: [PATCH 09/37] fix: clarify legacy compilers upgrade message - Updated the message to specify that the upgrade can only proceed if the compilers are no longer recognized as Puppet primary nodes. --- tasks/check_legacy_compilers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/check_legacy_compilers.rb b/tasks/check_legacy_compilers.rb index 65330ba9..7ca7220b 100755 --- a/tasks/check_legacy_compilers.rb +++ b/tasks/check_legacy_compilers.rb @@ -34,7 +34,7 @@ def execute! return unless !pinned_nodes.empty? puts 'The following legacy compilers are classified as Puppet primary nodes:' puts pinned_nodes.join(', ') - puts 'You will not be able to upgrade until these compilers are no longer classified as Puppet primary nodes.' + puts 'To continue with the upgrade, ensure that these compilers are no longer recognized as Puppet primary nodes.' end def https(port) From e566eb27de0c72c5f076bf56507553eecf4152d4 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 9 Sep 2024 12:36:28 +0100 Subject: [PATCH 10/37] chore(workflow): update test-upgrade-legacy.yaml for compiler versions Updated the compiler versions in the test-upgrade-legacy.yaml GitHub workflow from 2023.6.0 and 2023.7.0 to 2023.7.0 and 2023.8.0. Also, made formatting changes for consistency. --- .github/workflows/test-upgrade-legacy.yaml | 107 ++++++++------------- 1 file changed, 42 insertions(+), 65 deletions(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index a8fb954a..f3a1b1e8 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -1,80 +1,69 @@ --- -name: "Upgrade PE with one legacy compiler" - +name: Upgrade PE with one legacy compiler on: pull_request: paths: - - ".github/workflows/**/*" - - "spec/**/*" - - "lib/**/*" - - "tasks/**/*" - - "functions/**/*" - - "types/**/*" - - "plans/**/*" - - "hiera/**/*" - - "manifests/**/*" - - "templates/**/*" - - "files/**/*" - - "metadata.json" - - "Rakefile" - - "Gemfile" - - "provision.yaml" - - ".rspec" - - ".rubocop.yml" - - ".puppet-lint.rc" - - ".fixtures.yml" + - .github/workflows/**/* + - spec/**/* + - lib/**/* + - tasks/**/* + - functions/**/* + - types/**/* + - plans/**/* + - hiera/**/* + - manifests/**/* + - templates/**/* + - files/**/* + - metadata.json + - Rakefile + - Gemfile + - provision.yaml + - .rspec + - .rubocop.yml + - .puppet-lint.rc + - .fixtures.yml branches: [main] workflow_dispatch: - ssh-debugging: - description: "Boolean; whether or not to pause for ssh debugging" - required: true - default: "false" - + ssh-debugging: + description: Boolean; whether or not to pause for ssh debugging + required: true + default: 'false' jobs: test-install: - name: "PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}" + name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }} runs-on: ubuntu-20.04 env: BOLT_GEM: true BOLT_DISABLE_ANALYTICS: true - LANG: "en_US.UTF-8" + LANG: en_US.UTF-8 strategy: fail-fast: false matrix: - architecture: - - "large-with-two-compilers" - image: - - "almalinux-cloud/almalinux-8" - version: - - "2023.6.0" - to_version: - - "2023.7.0" - + architecture: [large-with-two-compilers] + image: [almalinux-cloud/almalinux-8] + version: [2023.7.0] + to_version: [2023.8.0] steps: - - name: "Start SSH session" + - name: Start SSH session if: ${{ github.event.inputs.ssh-debugging == 'true' }} uses: luchihoratiu/debug-via-ssh@main with: NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} SSH_PASS: ${{ secrets.SSH_PASS }} - - - name: "Checkout Source" + - name: Checkout Source uses: actions/checkout@v2 - - - name: "Activate Ruby 2.7" + - name: Activate Ruby 2.7 uses: ruby/setup-ruby@v1 with: - ruby-version: "2.7" + ruby-version: '2.7' bundler-cache: true - - - name: "Print bundle environment" + - name: Print bundle environment if: ${{ github.repository_owner == 'puppetlabs' }} run: | echo ::group::info:bundler bundle env echo ::endgroup:: - - - name: "Provision test cluster" + - name: Provision test cluster timeout-minutes: 15 run: | echo ::group::prepare @@ -84,7 +73,6 @@ jobs: echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config bundle exec rake spec_prep echo ::endgroup:: - echo ::group::provision bundle exec bolt plan run peadm_spec::provision_test_cluster \ --modulepath spec/fixtures/modules \ @@ -92,21 +80,17 @@ jobs: image=${{ matrix.image }} \ architecture=${{ matrix.architecture }} echo ::endgroup:: - echo ::group::info:request cat request.json || true; echo echo ::endgroup:: - echo ::group::info:inventory sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true echo ::endgroup:: - - name: Set up yq uses: frenck/action-setup-yq@v1 with: version: v4.30.5 - - - name: 'Install PE on test cluster' + - name: Install PE on test cluster timeout-minutes: 120 run: | bundle exec bolt plan run peadm_spec::install_test_cluster \ @@ -114,8 +98,7 @@ jobs: --modulepath spec/fixtures/modules \ architecture="large" \ version=${{ matrix.version }} - - - name: 'Wait as long as the file ${HOME}/pause file is present' + - name: Wait as long as the file ${HOME}/pause file is present if: ${{ always() && github.event.inputs.ssh-debugging == 'true' }} run: | while [ -f "${HOME}/pause" ] ; do @@ -123,22 +106,18 @@ jobs: sleep 60 done echo "${HOME}/pause absent, continuing workflow." - - - name: 'Convert one compiler to legacy' + - name: Convert one compiler to legacy timeout-minutes: 120 run: | primary=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .uri' spec/fixtures/litmus_inventory.yaml) compiler=$(yq '.groups[].targets[] | select(.vars.role == "compiler") | .uri' spec/fixtures/litmus_inventory.yaml | head -n 1) - bundle exec bolt plan run peadm::convert_compiler_to_legacy \ --inventoryfile spec/fixtures/litmus_inventory.yaml \ --modulepath spec/fixtures/modules \ --no-host-key-check \ primary_host=$primary \ legacy_hosts=$compiler - - - - name: 'Upgrade PE on test cluster' + - name: Upgrade PE on test cluster timeout-minutes: 120 run: | bundle exec bolt plan run peadm_spec::upgrade_test_cluster \ @@ -147,16 +126,14 @@ jobs: --no-host-key-check \ architecture="large" \ version=${{ matrix.to_version }} - - - name: "Tear down test cluster" + - name: Tear down test cluster if: ${{ always() }} continue-on-error: true - run: | + run: |- if [ -f spec/fixtures/litmus_inventory.yaml ]; then echo ::group::tear_down bundle exec rake 'litmus:tear_down' echo ::endgroup:: - echo ::group::info:request cat request.json || true; echo echo ::endgroup:: From 89d3acf8fc9bbd7308b3d614d341165442406119 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 9 Sep 2024 14:24:06 +0100 Subject: [PATCH 11/37] chore(workflow): add console_password to test-upgrade-legacy.yaml Added the console_password parameter to the test-upgrade-legacy.yaml GitHub workflow to ensure the console password is passed securely from secrets. --- .github/workflows/test-upgrade-legacy.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index f3a1b1e8..6df8f901 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -97,6 +97,7 @@ jobs: --inventoryfile spec/fixtures/litmus_inventory.yaml \ --modulepath spec/fixtures/modules \ architecture="large" \ + console_password=${{ secrets.CONSOLE_PASSWORD }} \ version=${{ matrix.version }} - name: Wait as long as the file ${HOME}/pause file is present if: ${{ always() && github.event.inputs.ssh-debugging == 'true' }} From 89243083f8a4fdd4b6be7790aaf99c1d3cab69ed Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 9 Sep 2024 16:57:44 +0100 Subject: [PATCH 12/37] refactor(plan): remove PuppetDB service stop commands in convert_compiler_to_legacy.pp Removed the commands to stop Puppet and PuppetDB services when the remove_pdb parameter is true in the convert_compiler_to_legacy plan. This simplifies the plan and avoids unnecessary service stops. --- plans/convert_compiler_to_legacy.pp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 4433cdc0..0cc654cd 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -21,11 +21,6 @@ getvar('cluster.params.compiler_hosts'), ]) - if $remove_pdb { - run_command('puppet resource service puppet ensure=stopped', $legacy_compiler_targets) - run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_compiler_targets) - } - apply($primary_target) { class { 'peadm::setup::node_manager_yaml': primary_host => $primary_target.peadm::certname(), @@ -36,7 +31,11 @@ } } - run_plan('peadm::update_compiler_extensions', compiler_hosts => $legacy_compiler_targets, primary_host => $primary_target, legacy => true) + run_plan('peadm::update_compiler_extensions', + compiler_hosts => $legacy_compiler_targets, + primary_host => $primary_target, + legacy => true + ) run_task('peadm::puppet_runonce', $legacy_compiler_targets) run_task('peadm::puppet_runonce', $primary_target) From b509f688a34b7ebd8d3ba19c30378849ded2b2bf Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 9 Sep 2024 17:54:36 +0100 Subject: [PATCH 13/37] fix(setup): correct variable reference for internal compiler pool address - Updated the variable reference for `internal_compiler_b_pool_address` and `internal_compiler_a_pool_address` in `peadm::setup::legacy_compiler_group` class. - Ensured the correct scope is used for the variables. - Added missing newline at the end of the file. - Added logic to stop and disable `pe-puppetdb` service if `remove_pdb` is true in `convert_compiler_to_legacy` plan. --- manifests/setup/legacy_compiler_group.pp | 7 ++++--- plans/convert_compiler_to_legacy.pp | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 24041e4a..5221141b 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -32,7 +32,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$peadm::setup::legacy_compiler_group::internal_compiler_b_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -54,7 +54,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$peadm::setup::legacy_compiler_group::internal_compiler_a_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -69,4 +69,5 @@ node_group { 'PE Compiler': rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], } -} \ No newline at end of file +} + diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 0cc654cd..64e96fec 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -21,6 +21,11 @@ getvar('cluster.params.compiler_hosts'), ]) + if $remove_pdb { + run_command('puppet resource service puppet ensure=stopped', $legacy_compiler_targets) + run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_compiler_targets) + } + apply($primary_target) { class { 'peadm::setup::node_manager_yaml': primary_host => $primary_target.peadm::certname(), From f7244a3d32bc32adb19e7aa4eaef6bd278e935ec Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Tue, 10 Sep 2024 16:46:51 +0100 Subject: [PATCH 14/37] Adding internal compiler address params to convert legacy compilers, and added trusted cert along with internal compiler address --- manifests/setup/legacy_compiler_group.pp | 13 +++++++++---- manifests/setup/node_manager.pp | 8 ++++++-- plans/convert_compiler_to_legacy.pp | 4 +++- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 5221141b..076773b4 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -1,6 +1,8 @@ # @api private class peadm::setup::legacy_compiler_group ( - String[1] $primary_host + String[1] $primary_host, + String[1] $internal_compiler_a_pool_address, + String[1] $internal_compiler_b_pool_address, ) { Node_group { purge_behavior => none, @@ -32,7 +34,9 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_b_pool_address].filter |$_| { $_ }, + # lint:endignore 'puppetdb_port' => [8081], }, }, @@ -54,7 +58,9 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_a_pool_address].filter |$_| { $_ }, + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_a_pool_address].filter |$_| { $_ }, + # lint:endignore 'puppetdb_port' => [8081], }, }, @@ -70,4 +76,3 @@ rule => ['and', ['=', ['trusted', 'extensions', peadm::oid('peadm_legacy_compiler')], 'false']], } } - diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index b6d5f096..e7074b63 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -229,7 +229,9 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_b_pool_address].filter |$_| { $_ }, + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_b_pool_address].filter |$_| { $_ }, + # lint:endignore 'puppetdb_port' => [8081], }, }, @@ -253,7 +255,9 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_a_pool_address].filter |$_| { $_ }, + # lint:ignore:single_quote_string_with_variables + 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_a_pool_address].filter |$_| { $_ }, + # lint:endignore 'puppetdb_port' => [8081], }, }, diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 64e96fec..fb7cf58c 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -32,7 +32,9 @@ } class { 'peadm::setup::legacy_compiler_group': - primary_host => $primary_target.peadm::certname(), + primary_host => $primary_target.peadm::certname(), + internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], + internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], } } From 123c1e64229d0b3f36859492d1a6dfcccc4220e3 Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Tue, 10 Sep 2024 17:32:13 +0100 Subject: [PATCH 15/37] Matching internal compiler address param types --- manifests/setup/legacy_compiler_group.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 076773b4..dcad98e5 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -1,8 +1,8 @@ # @api private class peadm::setup::legacy_compiler_group ( String[1] $primary_host, - String[1] $internal_compiler_a_pool_address, - String[1] $internal_compiler_b_pool_address, + Optional[String] $internal_compiler_a_pool_address = undef, + Optional[String] $internal_compiler_b_pool_address = undef, ) { Node_group { purge_behavior => none, From e3322769d00b5036984d0ce3c3dc7403925a93eb Mon Sep 17 00:00:00 2001 From: Neil Anderson Date: Tue, 10 Sep 2024 23:50:00 +0100 Subject: [PATCH 16/37] Adding require node_manager_yaml to legacy_compiler_group --- plans/convert_compiler_to_legacy.pp | 1 + 1 file changed, 1 insertion(+) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index fb7cf58c..5eb028f3 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -35,6 +35,7 @@ primary_host => $primary_target.peadm::certname(), internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], + require => Class['peadm::setup::node_manager_yaml'], } } From 97577d674bc22114a21f493dd376662b809791c5 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 11 Sep 2024 12:14:31 +0100 Subject: [PATCH 17/37] Introduced a puppet run before node_groups defs --- plans/convert_compiler_to_legacy.pp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 5eb028f3..4572e4fe 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -30,7 +30,11 @@ class { 'peadm::setup::node_manager_yaml': primary_host => $primary_target.peadm::certname(), } + } + + run_task('peadm::puppet_runonce', $primary_target) + apply($primary_target) { class { 'peadm::setup::legacy_compiler_group': primary_host => $primary_target.peadm::certname(), internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], From bd2acdc64fb8d3b1f410f2ea88b6b425f5142e78 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 11 Sep 2024 14:25:55 +0100 Subject: [PATCH 18/37] fix(setup): update puppetdb_host to use internal compiler pool addresses - Replaced `${trusted['certname']}` with `$internal_compiler_a_pool_address` and `$internal_compiler_b_pool_address` in `puppetdb_host` for `puppet_enterprise::profile::master` class. - Applied `.filter |$_| { $_ }` to ensure non-empty values. - Removed lint ignore comments for single quote string with variables. This change ensures that the correct internal compiler pool addresses are used for the `puppetdb_host` configuration. --- manifests/setup/legacy_compiler_group.pp | 12 +++--------- manifests/setup/node_manager.pp | 12 +++--------- 2 files changed, 6 insertions(+), 18 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index dcad98e5..870297ad 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -16,9 +16,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}'], - # lint:endignore + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -34,9 +32,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_b_pool_address].filter |$_| { $_ }, - # lint:endignore + 'puppetdb_host' => [$internal_compiler_b_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -58,9 +54,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_a_pool_address].filter |$_| { $_ }, - # lint:endignore + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index e7074b63..f74cb217 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -209,9 +209,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}'], - # lint:endignore + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -229,9 +227,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_b_pool_address].filter |$_| { $_ }, - # lint:endignore + 'puppetdb_host' => [$internal_compiler_b_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, @@ -255,9 +251,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - # lint:ignore:single_quote_string_with_variables - 'puppetdb_host' => ['${trusted[\'certname\']}', $internal_compiler_a_pool_address].filter |$_| { $_ }, - # lint:endignore + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, From f38f402c6157d1c8e10fea6abe26933c033c11f0 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 11 Sep 2024 14:45:28 +0100 Subject: [PATCH 19/37] docs: update upgrade_with_legacy_compilers.md with remove_pdb parameter info - Added information about the `remove_pdb` parameter to the documentation. - Updated `convert_compiler_to_legacy` plan to set `remove_pdb` default to `true`. This change provides clarity on how to retain PuppetDB service on converted compilers. --- documentation/upgrade_with_legacy_compilers.md | 2 ++ plans/convert_compiler_to_legacy.pp | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/upgrade_with_legacy_compilers.md b/documentation/upgrade_with_legacy_compilers.md index b33f4bf5..728253ec 100644 --- a/documentation/upgrade_with_legacy_compilers.md +++ b/documentation/upgrade_with_legacy_compilers.md @@ -40,6 +40,8 @@ bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.c The above will create the needed Node group and Classifier rules to make the compilers legacy. Also will add certificate extensions to those nodes. +If you want to keep puppetDB service on the converted compilers, you can do so by passing the `remove_pdb` parameter as `false`. + ### 4. Upgrade Puppet Enterprise After you have completed the above steps, you can proceed with the upgrade of Puppet Enterprise as usual using the puppetlabs-peadm module. There is no need to do the above ever again. diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 4572e4fe..97d25271 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -2,7 +2,7 @@ plan peadm::convert_compiler_to_legacy ( Peadm::SingleTargetSpec $primary_host, TargetSpec $legacy_hosts, - Boolean $remove_pdb = false, + Boolean $remove_pdb = true, ) { $primary_target = peadm::get_targets($primary_host, 1) $legacy_compiler_targets = peadm::get_targets($legacy_hosts) @@ -39,7 +39,6 @@ primary_host => $primary_target.peadm::certname(), internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], - require => Class['peadm::setup::node_manager_yaml'], } } From afff69ca94166455330829a2611b443ebbc28a13 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 11 Sep 2024 17:25:26 +0100 Subject: [PATCH 20/37] fix(workflow): correct indentation for ssh-debugging input in test-upgrade-legacy.yaml - Added missing `inputs` key for `ssh-debugging` under `workflow_dispatch`. This change ensures proper YAML structure for the workflow configuration. --- .github/workflows/test-upgrade-legacy.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index 6df8f901..63dcd0ab 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -24,10 +24,11 @@ on: - .fixtures.yml branches: [main] workflow_dispatch: - ssh-debugging: - description: Boolean; whether or not to pause for ssh debugging - required: true - default: 'false' + inputs: + ssh-debugging: + description: Boolean; whether or not to pause for ssh debugging + required: true + default: 'false' jobs: test-install: name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }} From 4c5bd6671853368d5bdeb608358f3b7bcd8bda0e Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 11 Sep 2024 17:20:11 +0100 Subject: [PATCH 21/37] chore(workflow): update test-upgrade-legacy.yaml name for clarity - Changed workflow name to "Upgrade PE with one legacy compiler (test)" for better identification. fix(convert_compiler_to_legacy): ensure proper handling of legacy compilers - Added architecture validation for supported architectures. - Split legacy compiler targets into groups A and B based on availability group or cluster. - Modified certificate extensions for legacy compilers. - Ensured proper stopping of PuppetDB service on legacy compilers when `remove_pdb` is true. - Updated node manager YAML and legacy compiler group setup to include internal compiler pool addresses. - Adjusted Puppet run tasks to include all compiler targets. These changes improve the handling and configuration of legacy compilers during the upgrade process. --- .github/workflows/test-upgrade-legacy.yaml | 2 +- plans/convert_compiler_to_legacy.pp | 86 +++++++++++++++++++--- 2 files changed, 76 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index 63dcd0ab..e5170d0c 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -1,5 +1,5 @@ --- -name: Upgrade PE with one legacy compiler +name: Upgrade PE with one legacy compiler (test) on: pull_request: paths: diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 97d25271..e57d4572 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -21,6 +21,78 @@ getvar('cluster.params.compiler_hosts'), ]) + # Ensure input valid for a supported architecture + $arch = peadm::assert_supported_architecture( + getvar('cluster.params.primary_host'), + getvar('cluster.params.replica_host'), + getvar('cluster.params.primary_postgresql_host'), + getvar('cluster.params.replica_postgresql_host'), + getvar('cluster.params.compiler_hosts'), + ) + + if $arch['disaster-recovery'] { + $legacy_compiler_a_targets = $legacy_compiler_targets.filter |$index,$target| { + $exts = $cert_extensions[$target.peadm::certname()] + if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { + $exts[peadm::oid('peadm_availability_group')] == 'A' + } + elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) { + $exts[peadm::oid('pp_cluster')] == 'A' + } + else { + $index % 2 == 0 + } + } + $legacy_compiler_b_targets = $legacy_compiler_targets.filter |$index,$target| { + $exts = $cert_extensions[$target.peadm::certname()] + if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { + $exts[peadm::oid('peadm_availability_group')] == 'B' + } + elsif ($exts[peadm::oid('pp_cluster')] in ['A', 'B']) { + $exts[peadm::oid('pp_cluster')] == 'B' + } + else { + $index % 2 != 0 + } + } + } else { + $legacy_compiler_a_targets = $legacy_compiler_targets + $legacy_compiler_b_targets = [] + } + + $compiler_targets = peadm::flatten_compact([getvar('cluster.params.compiler_hosts')]) + + wait([ + background('modify-compilers-certs') || { + run_plan('peadm::modify_certificate', $compiler_targets, + primary_host => $primary_target, + add_extensions => { + peadm::oid('peadm_legacy_compiler') => 'false', + }, + ) + }, + background('modify-compilers-a-certs') || { + run_plan('peadm::modify_certificate', $legacy_compiler_a_targets, + primary_host => $primary_target, + add_extensions => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'A', + peadm::oid('peadm_legacy_compiler') => 'true', + }, + ) + }, + background('modify-compilers-b-certs') || { + run_plan('peadm::modify_certificate', $legacy_compiler_b_targets, + primary_host => $primary_target, + add_extensions => { + peadm::oid('pp_auth_role') => 'pe_compiler', + peadm::oid('peadm_availability_group') => 'B', + peadm::oid('peadm_legacy_compiler') => 'true', + }, + ) + }, + ]) + if $remove_pdb { run_command('puppet resource service puppet ensure=stopped', $legacy_compiler_targets) run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_compiler_targets) @@ -29,26 +101,18 @@ apply($primary_target) { class { 'peadm::setup::node_manager_yaml': primary_host => $primary_target.peadm::certname(), - } - } - - run_task('peadm::puppet_runonce', $primary_target) + }, - apply($primary_target) { class { 'peadm::setup::legacy_compiler_group': primary_host => $primary_target.peadm::certname(), internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], + require => Class['peadm::setup::node_manager_yaml'], } } - run_plan('peadm::update_compiler_extensions', - compiler_hosts => $legacy_compiler_targets, - primary_host => $primary_target, - legacy => true - ) - run_task('peadm::puppet_runonce', $legacy_compiler_targets) + run_task('peadm::puppet_runonce', $compiler_targets) run_task('peadm::puppet_runonce', $primary_target) run_task('peadm::puppet_runonce', $all_targets) From 4b24dfbab421d125e9bbf39ac1c3da8d4f1dfe69 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Thu, 12 Sep 2024 11:39:43 +0100 Subject: [PATCH 22/37] feat(convert_compiler_to_legacy): update parameter types and improve formatting - Changed `remove_pdb` parameter type to `Optional[Boolean]` for better flexibility. - Improved alignment and formatting of parameters and variables for better readability. - Updated `$compiler_targets` assignment to use `peadm::get_targets` for consistency. --- plans/convert_compiler_to_legacy.pp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index e57d4572..e342419f 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -1,11 +1,11 @@ # @api private plan peadm::convert_compiler_to_legacy ( Peadm::SingleTargetSpec $primary_host, - TargetSpec $legacy_hosts, - Boolean $remove_pdb = true, + TargetSpec $legacy_hosts, + Optional[Boolean] $remove_pdb = true, ) { $primary_target = peadm::get_targets($primary_host, 1) - $legacy_compiler_targets = peadm::get_targets($legacy_hosts) + $legacy_compiler_targets = peadm::get_targets($legacy_hosts) $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value $error = getvar('cluster.error') @@ -60,7 +60,7 @@ $legacy_compiler_b_targets = [] } - $compiler_targets = peadm::flatten_compact([getvar('cluster.params.compiler_hosts')]) + $compiler_targets = peadm::get_targets(getvar('cluster.params.compiler_hosts')) wait([ background('modify-compilers-certs') || { @@ -101,7 +101,7 @@ apply($primary_target) { class { 'peadm::setup::node_manager_yaml': primary_host => $primary_target.peadm::certname(), - }, + } class { 'peadm::setup::legacy_compiler_group': primary_host => $primary_target.peadm::certname(), From 016bdd679eb42e09b19753a047e541fb71ecf1bc Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Thu, 12 Sep 2024 13:22:30 +0100 Subject: [PATCH 23/37] feat(workflow): enhance test-upgrade-legacy with SSH config and grouping - Added SSH configuration setup to ensure stable connections. - Grouped preparation and conversion steps for better readability in logs. --- .github/workflows/test-upgrade-legacy.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index e5170d0c..140bbc34 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -111,14 +111,23 @@ jobs: - name: Convert one compiler to legacy timeout-minutes: 120 run: | + echo ::group::prepare + mkdir -p $HOME/.ssh + echo 'Host *' > $HOME/.ssh/config + echo ' ServerAliveInterval 150' >> $HOME/.ssh/config + echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config + bundle exec rake spec_prep + echo ::endgroup:: primary=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .uri' spec/fixtures/litmus_inventory.yaml) compiler=$(yq '.groups[].targets[] | select(.vars.role == "compiler") | .uri' spec/fixtures/litmus_inventory.yaml | head -n 1) + echo ::group::convert_compiler_to_legacy bundle exec bolt plan run peadm::convert_compiler_to_legacy \ --inventoryfile spec/fixtures/litmus_inventory.yaml \ --modulepath spec/fixtures/modules \ --no-host-key-check \ primary_host=$primary \ legacy_hosts=$compiler + echo ::endgroup:: - name: Upgrade PE on test cluster timeout-minutes: 120 run: | From b93f4162ebafe8328dadbc7a23a378adc325a31b Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Thu, 12 Sep 2024 15:14:23 +0100 Subject: [PATCH 24/37] fix: handle undefined certname in convert_compiler_to_legacy plan - Updated `primary_host` to handle cases where `certname` is undefined. - Added conditional logic to default to `$primary_target` if `certname` is undefined. --- plans/convert_compiler_to_legacy.pp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index e342419f..3836e750 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -100,11 +100,17 @@ apply($primary_target) { class { 'peadm::setup::node_manager_yaml': - primary_host => $primary_target.peadm::certname(), + primary_host => $primary_target.peadm::certname() ? { + undef => $primary_target, + default => $primary_target.peadm::certname(), + }, } class { 'peadm::setup::legacy_compiler_group': - primary_host => $primary_target.peadm::certname(), + primary_host => $primary_target.peadm::certname() ? { + undef => $primary_target, + default => $primary_target.peadm::certname(), + }, internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], require => Class['peadm::setup::node_manager_yaml'], From f642fe8b73b506a4bc70a9c7a3017b12cb824842 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Thu, 12 Sep 2024 16:02:42 +0100 Subject: [PATCH 25/37] fix(workflow): downgrade test-upgrade-legacy version to 2021.7.9 - Changed the `version` in the test-upgrade-legacy workflow from 2023.7.0 to 2021.7.9. - Ensured compatibility with older versions for testing purposes. --- .github/workflows/test-upgrade-legacy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index 140bbc34..d74a4d53 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -42,7 +42,7 @@ jobs: matrix: architecture: [large-with-two-compilers] image: [almalinux-cloud/almalinux-8] - version: [2023.7.0] + version: [2021.7.9] to_version: [2023.8.0] steps: - name: Start SSH session From fb0122097b83092bdebc79a913ef9d706d0d9c71 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Thu, 12 Sep 2024 16:46:19 +0100 Subject: [PATCH 26/37] fix(workflow): update architecture in test-upgrade-legacy to large-with-dr - Changed the `architecture` in the test-upgrade-legacy workflow from `large-with-two-compilers` to `large-with-dr`. - Updated the `version` from `2021.7.9` to `2023.7.0`. --- .github/workflows/test-upgrade-legacy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index d74a4d53..8ba49118 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -40,9 +40,9 @@ jobs: strategy: fail-fast: false matrix: - architecture: [large-with-two-compilers] + architecture: [large-with-dr] image: [almalinux-cloud/almalinux-8] - version: [2021.7.9] + version: [2023.7.0] to_version: [2023.8.0] steps: - name: Start SSH session From 86ca580d3fc6205b71b7b0c589e3af9f9b331178 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 16 Sep 2024 11:56:06 +0100 Subject: [PATCH 27/37] Fix Conflicts --- REFERENCE.md | 586 +++++++++++++++++++++++++++++++-------------------- 1 file changed, 353 insertions(+), 233 deletions(-) diff --git a/REFERENCE.md b/REFERENCE.md index 5a6c2b90..bf4d04f9 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -8,131 +8,127 @@ #### Private Classes -- `peadm::setup::convert_node_manager`: Used during the peadm::convert plan -- `peadm::setup::convert_pre20197`: Defines configuration needed for converting PE 2018 -- `peadm::setup::legacy_compiler_group` -- `peadm::setup::node_manager`: Configures PEAdm's required node groups -- `peadm::setup::node_manager_yaml`: Set up the node_manager.yaml file in the temporary Bolt confdir +* `peadm::setup::convert_node_manager`: Used during the peadm::convert plan +* `peadm::setup::convert_pre20197`: Defines configuration needed for converting PE 2018 +* `peadm::setup::legacy_compiler_group` +* `peadm::setup::node_manager`: Configures PEAdm's required node groups +* `peadm::setup::node_manager_yaml`: Set up the node_manager.yaml file in the temporary Bolt confdir ### Functions -- [`peadm::assert_supported_architecture`](#peadm--assert_supported_architecture): Assert that the architecture given is a supported one -- [`peadm::assert_supported_bolt_version`](#peadm--assert_supported_bolt_version): Assert that the Bolt executable running PEAdm is a supported version -- [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm -- [`peadm::bolt_version`](#peadm--bolt_version) -- [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input -- [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts -- [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash -- [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors -- [`peadm::determine_status`](#peadm--determine_status): Produces a summarized hash of the given status data -- [`peadm::fail_on_transport`](#peadm--fail_on_transport): Fails if any nodes have the chosen transport. Useful for excluding PCP when it's not appopriate -- [`peadm::file_content_upload`](#peadm--file_content_upload) -- [`peadm::file_or_content`](#peadm--file_or_content) -- [`peadm::flatten_compact`](#peadm--flatten_compact) -- [`peadm::generate_pe_conf`](#peadm--generate_pe_conf): Generate a pe.conf file in JSON format -- [`peadm::get_pe_conf`](#peadm--get_pe_conf) -- [`peadm::get_targets`](#peadm--get_targets): Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - It returns an Array[Target -- [`peadm::migration_opts_default`](#peadm--migration_opts_default) -- [`peadm::node_manager_yaml_location`](#peadm--node_manager_yaml_location) -- [`peadm::oid`](#peadm--oid) -- [`peadm::plan_step`](#peadm--plan_step) -- [`peadm::recovery_opts_all`](#peadm--recovery_opts_all) -- [`peadm::recovery_opts_default`](#peadm--recovery_opts_default) -- [`peadm::update_pe_conf`](#peadm--update_pe_conf): Update the pe.conf file on a target with the provided hash -- [`peadm::wait_until_service_ready`](#peadm--wait_until_service_ready): A convenience function to help remember port numbers for services and handle running the wait_until_service_ready task +* [`peadm::assert_supported_architecture`](#peadm--assert_supported_architecture): Assert that the architecture given is a supported one +* [`peadm::assert_supported_bolt_version`](#peadm--assert_supported_bolt_version): Assert that the Bolt executable running PEAdm is a supported version +* [`peadm::assert_supported_pe_version`](#peadm--assert_supported_pe_version): Assert that the PE version given is supported by PEAdm +* [`peadm::bolt_version`](#peadm--bolt_version) +* [`peadm::certname`](#peadm--certname): Return the certname of the given target-like input +* [`peadm::check_version_and_known_hosts`](#peadm--check_version_and_known_hosts): Checks PE verison and warns about setting r10k_known_hosts +* [`peadm::convert_hash`](#peadm--convert_hash): converts two arrays into hash +* [`peadm::convert_status`](#peadm--convert_status): Transforms a value in a human readable status with or without colors +* [`peadm::determine_status`](#peadm--determine_status): Produces a summarized hash of the given status data +* [`peadm::fail_on_transport`](#peadm--fail_on_transport): Fails if any nodes have the chosen transport. Useful for excluding PCP when it's not appopriate +* [`peadm::file_content_upload`](#peadm--file_content_upload) +* [`peadm::file_or_content`](#peadm--file_or_content) +* [`peadm::flatten_compact`](#peadm--flatten_compact) +* [`peadm::generate_pe_conf`](#peadm--generate_pe_conf): Generate a pe.conf file in JSON format +* [`peadm::get_pe_conf`](#peadm--get_pe_conf) +* [`peadm::get_targets`](#peadm--get_targets): Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - It returns an Array[Target +* [`peadm::migration_opts_default`](#peadm--migration_opts_default) +* [`peadm::node_manager_yaml_location`](#peadm--node_manager_yaml_location) +* [`peadm::oid`](#peadm--oid) +* [`peadm::plan_step`](#peadm--plan_step) +* [`peadm::recovery_opts_all`](#peadm--recovery_opts_all) +* [`peadm::recovery_opts_default`](#peadm--recovery_opts_default) +* [`peadm::update_pe_conf`](#peadm--update_pe_conf): Update the pe.conf file on a target with the provided hash +* [`peadm::wait_until_service_ready`](#peadm--wait_until_service_ready): A convenience function to help remember port numbers for services and handle running the wait_until_service_ready task ### Data types -- [`Peadm::Known_hosts`](#Peadm--Known_hosts) -- [`Peadm::Ldap_config`](#Peadm--Ldap_config) -- [`Peadm::Pe_version`](#Peadm--Pe_version) -- [`Peadm::Pem`](#Peadm--Pem) -- [`Peadm::Recovery_opts`](#Peadm--Recovery_opts) -- [`Peadm::SingleTargetSpec`](#Peadm--SingleTargetSpec): A SingleTargetSpec represents any String, Target or single-element array of one or the other that can be passed to get_targets() to return an +* [`Peadm::Known_hosts`](#Peadm--Known_hosts) +* [`Peadm::Ldap_config`](#Peadm--Ldap_config) +* [`Peadm::Pe_version`](#Peadm--Pe_version) +* [`Peadm::Pem`](#Peadm--Pem) +* [`Peadm::Recovery_opts`](#Peadm--Recovery_opts) +* [`Peadm::SingleTargetSpec`](#Peadm--SingleTargetSpec): A SingleTargetSpec represents any String, Target or single-element array of one or the other that can be passed to get_targets() to return an ### Tasks -- [`agent_install`](#agent_install): Install the Puppet agent from a master -- [`backup_classification`](#backup_classification): A task to call the classification api and write to file -- [`cert_data`](#cert_data): Return certificate data related to the Puppet agent -- [`cert_valid_status`](#cert_valid_status): Check primary for valid state of a certificate -- [`classify_compilers`](#classify_compilers): Classify compilers as legacy or non-legacy -- [`code_manager`](#code_manager): Perform various code manager actions -- [`code_sync_status`](#code_sync_status): A task to confirm code is in sync accross the cluster for clusters with code manager configured -- [`divert_code_manager`](#divert_code_manager): Divert the code manager live-dir setting -- [`download`](#download): Download a file using curl -- [`enable_replica`](#enable_replica): Execute the enable replica puppet command -- [`filesize`](#filesize): Return the size of a file in bytes -- [`get_peadm_config`](#get_peadm_config): Run on a PE primary node to return the currently configured PEAdm parameters -- [`get_psql_version`](#get_psql_version): Run on a PE PSQL node to return the major version of the PSQL server currently installed -- [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output -- [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location -- [`mv`](#mv): Wrapper task for mv command -- [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string -- [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball -- [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console -- [`pe_uninstall`](#pe_uninstall): Uninstall Puppet Enterprise -- [`precheck`](#precheck): Return pre-check information about a system -- [`provision_replica`](#provision_replica): Execute the replica provision puppet command -- [`puppet_infra_upgrade`](#puppet_infra_upgrade): Execute the puppet infra upgrade command -- [`puppet_runonce`](#puppet_runonce): Run the Puppet agent one time -- [`rbac_token`](#rbac_token): Get and save an rbac token for the root user, admin rbac user -- [`read_file`](#read_file): Read the contents of a file -- [`reinstall_pe`](#reinstall_pe): Reinstall PE, only to be used to restore PE -- [`restore_classification`](#restore_classification): A short description of this task -- [`sign_csr`](#sign_csr): Submit a certificate signing request -- [`ssl_clean`](#ssl_clean): Clean an agent's certificate -- [`submit_csr`](#submit_csr): Submit a certificate signing request -- [`transform_classification_groups`](#transform_classification_groups): Transform the user groups from a source backup to a list of groups on the target server -- [`wait_until_service_ready`](#wait_until_service_ready): Return when the orchestrator service is healthy, or timeout after 15 seconds +* [`agent_install`](#agent_install): Install the Puppet agent from a master +* [`backup_classification`](#backup_classification): A task to call the classification api and write to file +* [`cert_data`](#cert_data): Return certificate data related to the Puppet agent +* [`cert_valid_status`](#cert_valid_status): Check primary for valid state of a certificate +* [`classify_compilers`](#classify_compilers): Classify compilers as legacy or non-legacy +* [`code_manager`](#code_manager): Perform various code manager actions +* [`code_sync_status`](#code_sync_status): A task to confirm code is in sync accross the cluster for clusters with code manager configured +* [`divert_code_manager`](#divert_code_manager): Divert the code manager live-dir setting +* [`download`](#download): Download a file using curl +* [`enable_replica`](#enable_replica): Execute the enable replica puppet command +* [`filesize`](#filesize): Return the size of a file in bytes +* [`get_peadm_config`](#get_peadm_config): Run on a PE primary node to return the currently configured PEAdm parameters +* [`get_psql_version`](#get_psql_version): Run on a PE PSQL node to return the major version of the PSQL server currently installed +* [`infrastatus`](#infrastatus): Runs puppet infra status and returns the output +* [`mkdir_p_file`](#mkdir_p_file): Create a file with the specified content at the specified location +* [`mv`](#mv): Wrapper task for mv command +* [`os_identification`](#os_identification): Return the operating system runnin gon the target as a string +* [`pe_install`](#pe_install): Install Puppet Enterprise from a tarball +* [`pe_ldap_config`](#pe_ldap_config): Set the ldap config in the PE console +* [`pe_uninstall`](#pe_uninstall): Uninstall Puppet Enterprise +* [`precheck`](#precheck): Return pre-check information about a system +* [`provision_replica`](#provision_replica): Execute the replica provision puppet command +* [`puppet_infra_upgrade`](#puppet_infra_upgrade): Execute the puppet infra upgrade command +* [`puppet_runonce`](#puppet_runonce): Run the Puppet agent one time +* [`rbac_token`](#rbac_token): Get and save an rbac token for the root user, admin rbac user +* [`read_file`](#read_file): Read the contents of a file +* [`reinstall_pe`](#reinstall_pe): Reinstall PE, only to be used to restore PE +* [`restore_classification`](#restore_classification): A short description of this task +* [`sign_csr`](#sign_csr): Submit a certificate signing request +* [`ssl_clean`](#ssl_clean): Clean an agent's certificate +* [`submit_csr`](#submit_csr): Submit a certificate signing request +* [`transform_classification_groups`](#transform_classification_groups): Transform the user groups from a source backup to a list of groups on the target server +* [`wait_until_service_ready`](#wait_until_service_ready): Return when the orchestrator service is healthy, or timeout after 15 seconds ### Plans #### Public Plans -- [`peadm::add_compiler`](#peadm--add_compiler): Add a new compiler to a PE architecture or replace an existing one with new configuration. -- [`peadm::add_database`](#peadm--add_database) -- [`peadm::add_replica`](#peadm--add_replica): Add or replace a replica host. - Supported use cases: - 1: Adding a replica to an existing primary. - 2: The existing replica is broken, we have a fresh new VM we want to provision the replica to. -- [`peadm::backup`](#peadm--backup): Backup puppet primary configuration -- [`peadm::backup_ca`](#peadm--backup_ca) -- [`peadm::convert`](#peadm--convert): Convert an existing PE cluster to a PEAdm-managed cluster -- [`peadm::install`](#peadm--install): Install a new PE cluster -- [`peadm::modify_certificate`](#peadm--modify_certificate): Modify the certificate of one or more targets -- [`peadm::restore`](#peadm--restore): Restore puppet primary configuration -- [`peadm::restore_ca`](#peadm--restore_ca) -- [`peadm::status`](#peadm--status): Return status information from one or more PE clusters in a table format -- [`peadm::upgrade`](#peadm--upgrade): Upgrade a PEAdm-managed cluster -- [`peadm::util::init_db_server`](#peadm--util--init_db_server) +* [`peadm::add_compiler`](#peadm--add_compiler): Add a new compiler to a PE architecture or replace an existing one with new configuration. +* [`peadm::add_database`](#peadm--add_database) +* [`peadm::add_replica`](#peadm--add_replica): Add or replace a replica host. +Supported use cases: +1: Adding a replica to an existing primary. +2: The existing replica is broken, we have a fresh new VM we want to provision the replica to. +* [`peadm::backup`](#peadm--backup): Backup puppet primary configuration +* [`peadm::backup_ca`](#peadm--backup_ca) +* [`peadm::convert`](#peadm--convert): Convert an existing PE cluster to a PEAdm-managed cluster +* [`peadm::install`](#peadm--install): Install a new PE cluster +* [`peadm::modify_certificate`](#peadm--modify_certificate): Modify the certificate of one or more targets +* [`peadm::restore`](#peadm--restore): Restore puppet primary configuration +* [`peadm::restore_ca`](#peadm--restore_ca) +* [`peadm::status`](#peadm--status): Return status information from one or more PE clusters in a table format +* [`peadm::upgrade`](#peadm--upgrade): Upgrade a PEAdm-managed cluster +* [`peadm::util::init_db_server`](#peadm--util--init_db_server) #### Private Plans -- `peadm::add_compiler`: Add a new compiler to a PE architecture or replace an existing one with new configuration. -- `peadm::add_replica`: Replace a replica host for a Standard or Large architecture. - Supported use cases: - 1: The existing replica is broken, we have a fresh new VM we want to provision the replica to. -- `peadm::convert_compiler_to_legacy` -- `peadm::misc::divert_code_manager`: This plan exists to account for a scenario where a PE XL -- `peadm::modify_cert_extensions` -- `peadm::subplans::component_install`: Install a new PEADM component -- `peadm::subplans::configure`: Configure first-time classification and DR setup -- `peadm::subplans::db_populate`: Destructively (re)populates a new or existing database with the contents or a known good source -- `peadm::subplans::install`: Perform initial installation of Puppet Enterprise Extra Large -- `peadm::subplans::modify_certificate` -- `peadm::subplans::prepare_agent` -- `peadm::uninstall`: Single-entry-point plan for uninstalling Puppet Enterprise -- `peadm::update_compiler_extensions` -- `peadm::util::code_sync_status` -- `peadm::util::copy_file` -- `peadm::util::db_disable_pglogical` -- `peadm::util::db_purge` -- `peadm::util::insert_csr_extension_requests` -- `peadm::util::retrieve_and_upload` -- `peadm::util::sanitize_pg_pe_conf` -- `peadm::util::update_classification`: Configure classification -- `peadm::util::update_db_setting`: Make updates to PuppetDB database settings +* `peadm::convert_compiler_to_legacy` +* `peadm::misc::divert_code_manager`: This plan exists to account for a scenario where a PE XL +* `peadm::modify_cert_extensions` +* `peadm::subplans::component_install`: Install a new PEADM component +* `peadm::subplans::configure`: Configure first-time classification and DR setup +* `peadm::subplans::db_populate`: Destructively (re)populates a new or existing database with the contents or a known good source +* `peadm::subplans::install`: Perform initial installation of Puppet Enterprise Extra Large +* `peadm::subplans::modify_certificate` +* `peadm::subplans::prepare_agent` +* `peadm::uninstall`: Single-entry-point plan for uninstalling Puppet Enterprise +* `peadm::update_compiler_extensions` +* `peadm::util::code_sync_status` +* `peadm::util::copy_file` +* `peadm::util::db_disable_pglogical` +* `peadm::util::db_purge` +* `peadm::util::insert_csr_extension_requests` +* `peadm::util::retrieve_and_upload` +* `peadm::util::sanitize_pg_pe_conf` +* `peadm::util::update_classification`: Configure classification +* `peadm::util::update_db_setting`: Make updates to PuppetDB database settings ## Functions @@ -152,24 +148,30 @@ Returns: `Hash` Data type: `TargetSpec` + + ##### `replica_host` Data type: `Variant[TargetSpec, Undef]` + + ##### `primary_postgresql_host` Data type: `Variant[TargetSpec, Undef]` + + ##### `replica_postgresql_host` Data type: `Variant[TargetSpec, Undef]` + + ##### `compiler_hosts` Data type: `Variant[TargetSpec, Undef]` -<<<<<<< HEAD -======= ##### `legacy_compilers` @@ -178,7 +180,6 @@ Data type: `Variant[TargetSpec, Undef]` ->>>>>>> d6467f9 ((PE-38770) Install Plan accepts legacy_compilers key (#474)) ### `peadm::assert_supported_bolt_version` Type: Puppet Language @@ -217,10 +218,14 @@ version number to check Data type: `String` + + ##### `permit_unsafe_versions` Data type: `Boolean` + + ### `peadm::bolt_version` Type: Ruby 4.x API @@ -246,14 +251,12 @@ is its certname. For strings, the certname is equal to the string. Undef input returns undef. #### `peadm::certname(Variant[Target, - String, Undef, Array[Target,1,1], Array[String,1,1], Array[Undef,1,1], - -Array[Any,0,0]] $target)` + Array[Any,0,0]] $target)` This function accepts a variety of data types which could represent single targets, and returns the certname corresponding to the input. @@ -279,6 +282,8 @@ Variant[Target, Array[Any,0,0]] ``` + + ### `peadm::check_version_and_known_hosts` Type: Puppet Language @@ -315,14 +320,20 @@ The r10k_known_hosts parameter Data type: `String` + + ##### `target_version` Data type: `String` + + ##### `r10k_known_hosts` Data type: `Optional[Peadm::Known_hosts]` + + ### `peadm::convert_hash` Type: Puppet Language @@ -604,14 +615,20 @@ Returns: `Any` Data type: `TargetSpec` + + ##### `transport` Data type: `String` + + ##### `message` Data type: `String` + + ### `peadm::file_content_upload` Type: Ruby 4.x API @@ -628,14 +645,20 @@ Returns: `Any` Data type: `String[1]` + + ##### `destination` Data type: `String[1]` + + ##### `*targets` Data type: `TargetOrTargets` + + ### `peadm::file_or_content` Type: Puppet Language @@ -652,14 +675,20 @@ Returns: `Any` Data type: `String` + + ##### `file` Data type: `Variant[String, Undef]` + + ##### `content` Data type: `Variant[String, Undef]` + + ### `peadm::flatten_compact` Type: Puppet Language @@ -676,6 +705,8 @@ Returns: `Any` Data type: `Array` + + ### `peadm::generate_pe_conf` Type: Puppet Language @@ -711,23 +742,23 @@ Returns: `Any` Data type: `Target` + + ### `peadm::get_targets` Type: Puppet Language Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - -- It returns an Array[Target, 1, 0], rather than a Target -- It will accept undef and return [ ]. + - It returns an Array[Target, 1, 0], rather than a Target + - It will accept undef and return [ ]. #### `peadm::get_targets(Variant[TargetSpec, Undef] $spec, Optional[Integer[1,1]] $count = undef)` Accept undef or a SingleTargetSpec, and return an Array[Target, 1, 0]. This differs from get_target() in that: - -- It returns an Array[Target, 1, 0], rather than a Target -- It will accept undef and return [ ]. + - It returns an Array[Target, 1, 0], rather than a Target + - It will accept undef and return [ ]. Returns: `Any` @@ -735,10 +766,14 @@ Returns: `Any` Data type: `Variant[TargetSpec, Undef]` + + ##### `count` Data type: `Optional[Integer[1,1]]` + + ### `peadm::migration_opts_default` Type: Puppet Language @@ -779,6 +814,8 @@ Returns: `Any` Data type: `String` + + ### `peadm::plan_step` Type: Ruby 4.x API @@ -795,10 +832,14 @@ Returns: `Any` Data type: `String` + + ##### `&block` Data type: `Callable` + + ### `peadm::recovery_opts_all` Type: Puppet Language @@ -865,10 +906,14 @@ Returns: `Any` Data type: `String` + + ##### `target` Data type: `TargetSpec` + + ## Data types ### `Peadm::Known_hosts` @@ -956,9 +1001,9 @@ Struct[{ ### `Peadm::SingleTargetSpec` A SingleTargetSpec represents any String, Target or single-element array of -one or the other that can be passed to get*targets() to return an +one or the other that can be passed to get_targets() to return an Array[Target, 1, 1]. This is a constrained type variant of -Boltlib::TargetSpec for use when a \_single* target is valid, but multiple +Boltlib::TargetSpec for use when a _single_ target is valid, but multiple targets are not. Alias of `Variant[Pattern[/\A[^[:space:],]+\z/], Target, Array[Peadm::SingleTargetSpec, 1, 1]]` @@ -1543,17 +1588,17 @@ Add a new compiler to a PE architecture or replace an existing one with new conf The following parameters are available in the `peadm::add_compiler` plan: -- [`avail_group_letter`](#-peadm--add_compiler--avail_group_letter) -- [`compiler_host`](#-peadm--add_compiler--compiler_host) -- [`dns_alt_names`](#-peadm--add_compiler--dns_alt_names) -- [`primary_host`](#-peadm--add_compiler--primary_host) -- [`primary_postgresql_host`](#-peadm--add_compiler--primary_postgresql_host) +* [`avail_group_letter`](#-peadm--add_compiler--avail_group_letter) +* [`compiler_host`](#-peadm--add_compiler--compiler_host) +* [`dns_alt_names`](#-peadm--add_compiler--dns_alt_names) +* [`primary_host`](#-peadm--add_compiler--primary_host) +* [`primary_postgresql_host`](#-peadm--add_compiler--primary_postgresql_host) ##### `avail_group_letter` Data type: `Enum['A', 'B']` -\_ Either A or B; whichever of the two letter designations the compiler is being assigned to +_ Either A or B; whichever of the two letter designations the compiler is being assigned to Default value: `'A'` @@ -1561,13 +1606,13 @@ Default value: `'A'` Data type: `Peadm::SingleTargetSpec` -\_ The hostname and certname of the new compiler +_ The hostname and certname of the new compiler ##### `dns_alt_names` Data type: `Optional[String[1]]` -\_ A comma_separated list of DNS alt names for the compiler +_ A comma_separated list of DNS alt names for the compiler Default value: `undef` @@ -1575,13 +1620,13 @@ Default value: `undef` Data type: `Peadm::SingleTargetSpec` -\_ The hostname and certname of the primary Puppet server +_ The hostname and certname of the primary Puppet server ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` -\_ The hostname and certname of the PE-PostgreSQL server with availability group $avail_group_letter +_ The hostname and certname of the PE-PostgreSQL server with availability group $avail_group_letter Default value: `undef` @@ -1593,23 +1638,29 @@ The peadm::add_database class. The following parameters are available in the `peadm::add_database` plan: -- [`targets`](#-peadm--add_database--targets) -- [`primary_host`](#-peadm--add_database--primary_host) -- [`mode`](#-peadm--add_database--mode) -- [`begin_at_step`](#-peadm--add_database--begin_at_step) +* [`targets`](#-peadm--add_database--targets) +* [`primary_host`](#-peadm--add_database--primary_host) +* [`mode`](#-peadm--add_database--mode) +* [`begin_at_step`](#-peadm--add_database--begin_at_step) ##### `targets` Data type: `Peadm::SingleTargetSpec` + + ##### `primary_host` Data type: `Peadm::SingleTargetSpec` + + ##### `mode` Data type: `Optional[Enum['init', 'pair']]` + + Default value: `undef` ##### `begin_at_step` @@ -1626,6 +1677,8 @@ Optional[Enum[ 'finalize']] ``` + + Default value: `undef` ### `peadm::add_replica` @@ -1639,10 +1692,10 @@ Supported use cases: The following parameters are available in the `peadm::add_replica` plan: -- [`primary_host`](#-peadm--add_replica--primary_host) -- [`replica_host`](#-peadm--add_replica--replica_host) -- [`replica_postgresql_host`](#-peadm--add_replica--replica_postgresql_host) -- [`token_file`](#-peadm--add_replica--token_file) +* [`primary_host`](#-peadm--add_replica--primary_host) +* [`replica_host`](#-peadm--add_replica--replica_host) +* [`replica_postgresql_host`](#-peadm--add_replica--replica_postgresql_host) +* [`token_file`](#-peadm--add_replica--token_file) ##### `primary_host` @@ -1661,7 +1714,7 @@ Data type: `Peadm::SingleTargetSpec` Data type: `Optional[Peadm::SingleTargetSpec]` - The hostname and certname of the host with the replica PE-PosgreSQL database. - Can be a separate host in an XL architecture, or undef in Standard or Large. +Can be a separate host in an XL architecture, or undef in Standard or Large. Default value: `undef` @@ -1679,7 +1732,7 @@ Backup puppet primary configuration #### Examples -##### +##### ```puppet bolt plan run peadm::backup -t primary1.example.com @@ -1689,10 +1742,10 @@ bolt plan run peadm::backup -t primary1.example.com The following parameters are available in the `peadm::backup` plan: -- [`targets`](#-peadm--backup--targets) -- [`backup_type`](#-peadm--backup--backup_type) -- [`backup`](#-peadm--backup--backup) -- [`output_directory`](#-peadm--backup--output_directory) +* [`targets`](#-peadm--backup--targets) +* [`backup_type`](#-peadm--backup--backup_type) +* [`backup`](#-peadm--backup--backup) +* [`output_directory`](#-peadm--backup--output_directory) ##### `targets` @@ -1732,17 +1785,21 @@ The peadm::backup_ca class. The following parameters are available in the `peadm::backup_ca` plan: -- [`target`](#-peadm--backup_ca--target) -- [`output_directory`](#-peadm--backup_ca--output_directory) +* [`target`](#-peadm--backup_ca--target) +* [`output_directory`](#-peadm--backup_ca--output_directory) ##### `target` Data type: `Peadm::SingleTargetSpec` + + ##### `output_directory` Data type: `Optional[String]` + + Default value: `'/tmp'` ### `peadm::convert` @@ -1755,18 +1812,6 @@ management using PEAdm. The following parameters are available in the `peadm::convert` plan: -<<<<<<< HEAD -- [`primary_host`](#-peadm--convert--primary_host) -- [`replica_host`](#-peadm--convert--replica_host) -- [`compiler_hosts`](#-peadm--convert--compiler_hosts) -- [`primary_postgresql_host`](#-peadm--convert--primary_postgresql_host) -- [`replica_postgresql_host`](#-peadm--convert--replica_postgresql_host) -- [`compiler_pool_address`](#-peadm--convert--compiler_pool_address) -- [`internal_compiler_a_pool_address`](#-peadm--convert--internal_compiler_a_pool_address) -- [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) -- [`dns_alt_names`](#-peadm--convert--dns_alt_names) -- [`begin_at_step`](#-peadm--convert--begin_at_step) -======= * [`primary_host`](#-peadm--convert--primary_host) * [`replica_host`](#-peadm--convert--replica_host) * [`compiler_hosts`](#-peadm--convert--compiler_hosts) @@ -1778,22 +1823,27 @@ The following parameters are available in the `peadm::convert` plan: * [`internal_compiler_b_pool_address`](#-peadm--convert--internal_compiler_b_pool_address) * [`dns_alt_names`](#-peadm--convert--dns_alt_names) * [`begin_at_step`](#-peadm--convert--begin_at_step) ->>>>>>> 671839c ((PE-38771) Convert plan accepts legacy compilers key in params.json (#476)) ##### `primary_host` Data type: `Peadm::SingleTargetSpec` + + ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` + + Default value: `undef` ##### `legacy_compilers` @@ -1808,36 +1858,48 @@ Default value: `undef` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `compiler_pool_address` Data type: `String` + + Default value: `$primary_host` ##### `internal_compiler_a_pool_address` Data type: `Optional[String]` + + Default value: `undef` ##### `internal_compiler_b_pool_address` Data type: `Optional[String]` + + Default value: `undef` ##### `dns_alt_names` Data type: `Array[String]` + + Default value: `[]` ##### `begin_at_step` @@ -1852,6 +1914,8 @@ Optional[Enum[ 'finalize']] ``` + + Default value: `undef` ### `peadm::install` @@ -1862,36 +1926,6 @@ Install a new PE cluster The following parameters are available in the `peadm::install` plan: -<<<<<<< HEAD -- [`compiler_pool_address`](#-peadm--install--compiler_pool_address) -- [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) -- [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) -- [`pe_installer_source`](#-peadm--install--pe_installer_source) -- [`ldap_config`](#-peadm--install--ldap_config) -- [`final_agent_state`](#-peadm--install--final_agent_state) -- [`stagingdir`](#-peadm--install--stagingdir) -- [`uploaddir`](#-peadm--install--uploaddir) -- [`primary_host`](#-peadm--install--primary_host) -- [`replica_host`](#-peadm--install--replica_host) -- [`compiler_hosts`](#-peadm--install--compiler_hosts) -- [`primary_postgresql_host`](#-peadm--install--primary_postgresql_host) -- [`replica_postgresql_host`](#-peadm--install--replica_postgresql_host) -- [`console_password`](#-peadm--install--console_password) -- [`version`](#-peadm--install--version) -- [`dns_alt_names`](#-peadm--install--dns_alt_names) -- [`pe_conf_data`](#-peadm--install--pe_conf_data) -- [`code_manager_auto_configure`](#-peadm--install--code_manager_auto_configure) -- [`r10k_remote`](#-peadm--install--r10k_remote) -- [`r10k_private_key_file`](#-peadm--install--r10k_private_key_file) -- [`r10k_private_key_content`](#-peadm--install--r10k_private_key_content) -- [`r10k_known_hosts`](#-peadm--install--r10k_known_hosts) -- [`deploy_environment`](#-peadm--install--deploy_environment) -- [`license_key_file`](#-peadm--install--license_key_file) -- [`license_key_content`](#-peadm--install--license_key_content) -- [`download_mode`](#-peadm--install--download_mode) -- [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) -- [`token_lifetime`](#-peadm--install--token_lifetime) -======= * [`compiler_pool_address`](#-peadm--install--compiler_pool_address) * [`internal_compiler_a_pool_address`](#-peadm--install--internal_compiler_a_pool_address) * [`internal_compiler_b_pool_address`](#-peadm--install--internal_compiler_b_pool_address) @@ -1921,7 +1955,6 @@ The following parameters are available in the `peadm::install` plan: * [`download_mode`](#-peadm--install--download_mode) * [`permit_unsafe_versions`](#-peadm--install--permit_unsafe_versions) * [`token_lifetime`](#-peadm--install--token_lifetime) ->>>>>>> d6467f9 ((PE-38770) Install Plan accepts legacy_compilers key (#474)) ##### `compiler_pool_address` @@ -2005,16 +2038,22 @@ Default value: `undef` Data type: `Peadm::SingleTargetSpec` + + ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` + + Default value: `undef` ##### `legacy_compilers` @@ -2029,100 +2068,134 @@ Default value: `undef` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `console_password` Data type: `String` + + ##### `version` Data type: `Peadm::Pe_version` + + Default value: `'2021.7.9'` ##### `dns_alt_names` Data type: `Optional[Array[String]]` + + Default value: `undef` ##### `pe_conf_data` Data type: `Optional[Hash]` + + Default value: `{}` ##### `code_manager_auto_configure` Data type: `Optional[Boolean]` + + Default value: `undef` ##### `r10k_remote` Data type: `Optional[String]` + + Default value: `undef` ##### `r10k_private_key_file` Data type: `Optional[String]` + + Default value: `undef` ##### `r10k_private_key_content` Data type: `Optional[Peadm::Pem]` + + Default value: `undef` ##### `r10k_known_hosts` Data type: `Optional[Peadm::Known_hosts]` + + Default value: `undef` ##### `deploy_environment` Data type: `Optional[String]` + + Default value: `undef` ##### `license_key_file` Data type: `Optional[String]` + + Default value: `undef` ##### `license_key_content` Data type: `Optional[String]` + + Default value: `undef` ##### `download_mode` Data type: `Enum['direct', 'bolthost']` + + Default value: `'bolthost'` ##### `permit_unsafe_versions` Data type: `Boolean` + + Default value: `false` ##### `token_lifetime` Data type: `String` + + Default value: `'1y'` ### `peadm::modify_certificate` @@ -2134,43 +2207,55 @@ setting DNS alternative names. The following parameters are available in the `peadm::modify_certificate` plan: -- [`targets`](#-peadm--modify_certificate--targets) -- [`primary_host`](#-peadm--modify_certificate--primary_host) -- [`add_extensions`](#-peadm--modify_certificate--add_extensions) -- [`remove_extensions`](#-peadm--modify_certificate--remove_extensions) -- [`dns_alt_names`](#-peadm--modify_certificate--dns_alt_names) -- [`force_regenerate`](#-peadm--modify_certificate--force_regenerate) +* [`targets`](#-peadm--modify_certificate--targets) +* [`primary_host`](#-peadm--modify_certificate--primary_host) +* [`add_extensions`](#-peadm--modify_certificate--add_extensions) +* [`remove_extensions`](#-peadm--modify_certificate--remove_extensions) +* [`dns_alt_names`](#-peadm--modify_certificate--dns_alt_names) +* [`force_regenerate`](#-peadm--modify_certificate--force_regenerate) ##### `targets` Data type: `TargetSpec` + + ##### `primary_host` Data type: `Peadm::SingleTargetSpec` + + ##### `add_extensions` Data type: `Hash` + + Default value: `{}` ##### `remove_extensions` Data type: `Array` + + Default value: `[]` ##### `dns_alt_names` Data type: `Optional[Array]` + + Default value: `undef` ##### `force_regenerate` Data type: `Boolean` + + Default value: `false` ### `peadm::restore` @@ -2179,7 +2264,7 @@ Restore puppet primary configuration #### Examples -##### +##### ```puppet bolt plan run peadm::restore -t primary1.example.com input_file=/tmp/peadm-backup.tar.gz @@ -2189,10 +2274,10 @@ bolt plan run peadm::restore -t primary1.example.com input_file=/tmp/peadm-backu The following parameters are available in the `peadm::restore` plan: -- [`targets`](#-peadm--restore--targets) -- [`restore_type`](#-peadm--restore--restore_type) -- [`restore`](#-peadm--restore--restore) -- [`input_file`](#-peadm--restore--input_file) +* [`targets`](#-peadm--restore--targets) +* [`restore_type`](#-peadm--restore--restore_type) +* [`restore`](#-peadm--restore--restore) +* [`input_file`](#-peadm--restore--input_file) ##### `targets` @@ -2230,22 +2315,28 @@ The peadm::restore_ca class. The following parameters are available in the `peadm::restore_ca` plan: -- [`target`](#-peadm--restore_ca--target) -- [`file_path`](#-peadm--restore_ca--file_path) -- [`recovery_directory`](#-peadm--restore_ca--recovery_directory) +* [`target`](#-peadm--restore_ca--target) +* [`file_path`](#-peadm--restore_ca--file_path) +* [`recovery_directory`](#-peadm--restore_ca--recovery_directory) ##### `target` Data type: `Peadm::SingleTargetSpec` + + ##### `file_path` Data type: `String` + + ##### `recovery_directory` Data type: `Optional[String]` + + Default value: `'/tmp/peadm_recovery'` ### `peadm::status` @@ -2264,11 +2355,11 @@ peadm::status($targets, 'table', true, true) The following parameters are available in the `peadm::status` plan: -- [`targets`](#-peadm--status--targets) -- [`format`](#-peadm--status--format) -- [`summarize`](#-peadm--status--summarize) -- [`verbose`](#-peadm--status--verbose) -- [`colors`](#-peadm--status--colors) +* [`targets`](#-peadm--status--targets) +* [`format`](#-peadm--status--format) +* [`summarize`](#-peadm--status--summarize) +* [`verbose`](#-peadm--status--verbose) +* [`colors`](#-peadm--status--colors) ##### `targets` @@ -2316,24 +2407,24 @@ Upgrade a PEAdm-managed cluster The following parameters are available in the `peadm::upgrade` plan: -- [`compiler_pool_address`](#-peadm--upgrade--compiler_pool_address) -- [`internal_compiler_a_pool_address`](#-peadm--upgrade--internal_compiler_a_pool_address) -- [`internal_compiler_b_pool_address`](#-peadm--upgrade--internal_compiler_b_pool_address) -- [`pe_installer_source`](#-peadm--upgrade--pe_installer_source) -- [`final_agent_state`](#-peadm--upgrade--final_agent_state) -- [`r10k_known_hosts`](#-peadm--upgrade--r10k_known_hosts) -- [`stagingdir`](#-peadm--upgrade--stagingdir) -- [`uploaddir`](#-peadm--upgrade--uploaddir) -- [`primary_host`](#-peadm--upgrade--primary_host) -- [`replica_host`](#-peadm--upgrade--replica_host) -- [`compiler_hosts`](#-peadm--upgrade--compiler_hosts) -- [`primary_postgresql_host`](#-peadm--upgrade--primary_postgresql_host) -- [`replica_postgresql_host`](#-peadm--upgrade--replica_postgresql_host) -- [`version`](#-peadm--upgrade--version) -- [`token_file`](#-peadm--upgrade--token_file) -- [`download_mode`](#-peadm--upgrade--download_mode) -- [`permit_unsafe_versions`](#-peadm--upgrade--permit_unsafe_versions) -- [`begin_at_step`](#-peadm--upgrade--begin_at_step) +* [`compiler_pool_address`](#-peadm--upgrade--compiler_pool_address) +* [`internal_compiler_a_pool_address`](#-peadm--upgrade--internal_compiler_a_pool_address) +* [`internal_compiler_b_pool_address`](#-peadm--upgrade--internal_compiler_b_pool_address) +* [`pe_installer_source`](#-peadm--upgrade--pe_installer_source) +* [`final_agent_state`](#-peadm--upgrade--final_agent_state) +* [`r10k_known_hosts`](#-peadm--upgrade--r10k_known_hosts) +* [`stagingdir`](#-peadm--upgrade--stagingdir) +* [`uploaddir`](#-peadm--upgrade--uploaddir) +* [`primary_host`](#-peadm--upgrade--primary_host) +* [`replica_host`](#-peadm--upgrade--replica_host) +* [`compiler_hosts`](#-peadm--upgrade--compiler_hosts) +* [`primary_postgresql_host`](#-peadm--upgrade--primary_postgresql_host) +* [`replica_postgresql_host`](#-peadm--upgrade--replica_postgresql_host) +* [`version`](#-peadm--upgrade--version) +* [`token_file`](#-peadm--upgrade--token_file) +* [`download_mode`](#-peadm--upgrade--download_mode) +* [`permit_unsafe_versions`](#-peadm--upgrade--permit_unsafe_versions) +* [`begin_at_step`](#-peadm--upgrade--begin_at_step) ##### `compiler_pool_address` @@ -2417,52 +2508,70 @@ Default value: `'/tmp'` Data type: `Peadm::SingleTargetSpec` + + ##### `replica_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `compiler_hosts` Data type: `Optional[TargetSpec]` + + Default value: `undef` ##### `primary_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `replica_postgresql_host` Data type: `Optional[Peadm::SingleTargetSpec]` + + Default value: `undef` ##### `version` Data type: `Optional[Peadm::Pe_version]` + + Default value: `undef` ##### `token_file` Data type: `Optional[String]` + + Default value: `undef` ##### `download_mode` Data type: `Enum[direct,bolthost]` + + Default value: `'bolthost'` ##### `permit_unsafe_versions` Data type: `Boolean` + + Default value: `false` ##### `begin_at_step` @@ -2479,6 +2588,8 @@ Optional[Enum[ 'finalize']] ``` + + Default value: `undef` ### `peadm::util::init_db_server` @@ -2489,29 +2600,38 @@ The peadm::util::init_db_server class. The following parameters are available in the `peadm::util::init_db_server` plan: -- [`db_host`](#-peadm--util--init_db_server--db_host) -- [`install_pe`](#-peadm--util--init_db_server--install_pe) -- [`pe_version`](#-peadm--util--init_db_server--pe_version) -- [`pe_platform`](#-peadm--util--init_db_server--pe_platform) +* [`db_host`](#-peadm--util--init_db_server--db_host) +* [`install_pe`](#-peadm--util--init_db_server--install_pe) +* [`pe_version`](#-peadm--util--init_db_server--pe_version) +* [`pe_platform`](#-peadm--util--init_db_server--pe_platform) ##### `db_host` Data type: `String[1]` + + ##### `install_pe` Data type: `Boolean` + + Default value: `false` ##### `pe_version` Data type: `String[1]` + + Default value: `'2023.5.0'` ##### `pe_platform` Data type: `String[1]` + + Default value: `'el-8-x86_64'` + From 0ae01f3c11fb84ef258a92cdd83ff41a43fc5afd Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 16 Sep 2024 12:02:52 +0100 Subject: [PATCH 28/37] feat(plan): run puppet agent on all nodes after installation - Added a task to run the puppet agent on all nodes after the installation plan. --- spec/acceptance/peadm_spec/plans/install_test_cluster.pp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/acceptance/peadm_spec/plans/install_test_cluster.pp b/spec/acceptance/peadm_spec/plans/install_test_cluster.pp index d7693e24..7ed406bc 100644 --- a/spec/acceptance/peadm_spec/plans/install_test_cluster.pp +++ b/spec/acceptance/peadm_spec/plans/install_test_cluster.pp @@ -79,5 +79,8 @@ $install_result = run_plan('peadm::install', $arch_params + $common_params) + # Run puppet agent on all nodes + run_task('peadm::puppet_runonce', $t) + return($install_result) } From b30996c8e757fdb0d73d7154edbf587e438892b8 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 16 Sep 2024 12:58:24 +0100 Subject: [PATCH 29/37] fix(plan): remove redundant puppet agent run task - Removed the redundant task to run the puppet agent on all nodes after installation. - Simplified the install_test_cluster plan by eliminating unnecessary steps. --- .github/workflows/test-upgrade-legacy.yaml | 4 ++-- spec/acceptance/peadm_spec/plans/install_test_cluster.pp | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml index 8ba49118..2edcd265 100644 --- a/.github/workflows/test-upgrade-legacy.yaml +++ b/.github/workflows/test-upgrade-legacy.yaml @@ -97,7 +97,7 @@ jobs: bundle exec bolt plan run peadm_spec::install_test_cluster \ --inventoryfile spec/fixtures/litmus_inventory.yaml \ --modulepath spec/fixtures/modules \ - architecture="large" \ + architecture=${{ matrix.architecture }} \ console_password=${{ secrets.CONSOLE_PASSWORD }} \ version=${{ matrix.version }} - name: Wait as long as the file ${HOME}/pause file is present @@ -135,7 +135,7 @@ jobs: --inventoryfile spec/fixtures/litmus_inventory.yaml \ --modulepath spec/fixtures/modules \ --no-host-key-check \ - architecture="large" \ + architecture=${{ matrix.architecture }} \ version=${{ matrix.to_version }} - name: Tear down test cluster if: ${{ always() }} diff --git a/spec/acceptance/peadm_spec/plans/install_test_cluster.pp b/spec/acceptance/peadm_spec/plans/install_test_cluster.pp index 7ed406bc..d7693e24 100644 --- a/spec/acceptance/peadm_spec/plans/install_test_cluster.pp +++ b/spec/acceptance/peadm_spec/plans/install_test_cluster.pp @@ -79,8 +79,5 @@ $install_result = run_plan('peadm::install', $arch_params + $common_params) - # Run puppet agent on all nodes - run_task('peadm::puppet_runonce', $t) - return($install_result) } From 79dc961bd1fa1e42cf2060fb6737898f67aa4a06 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 16 Sep 2024 14:16:14 +0100 Subject: [PATCH 30/37] feat(plans): gather certificate extension information in convert_compiler_to_legacy plan - Added a step to gather certificate extension information from all systems using the `peadm::cert_data` task. - This information is used to filter legacy compiler targets based on availability group. This enhancement improves the accuracy of the conversion process by ensuring that the correct certificate data is used. --- README.md | 59 ++++++++++++++--------------- plans/convert_compiler_to_legacy.pp | 4 ++ 2 files changed, 33 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index ef33203a..9efda46b 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ You can use PEADM to deploy and manage PE installations for standard, large, and #### Table of contents - [Puppet Enterprise Administration Module (PEADM)](#puppet-enterprise-pe-administration-adm-module) - - [Table of contents](#table-of-contents) + - [Table of contents](#table-of-contents) - [Support](#support) - [Overview](#overview) - [What PEADM affects](#what-peadm-affects) @@ -24,7 +24,6 @@ You can use PEADM to deploy and manage PE installations for standard, large, and PEADM is a supported PE module. If you are a PE customer with the standard or premium support service, you can contact [Support](https://portal.perforce.com/s/topic/0TO4X000000DbNgWAK/puppet) or your Technical Account Manager for assistance. - ## Overview This is the standard workflow for installing PEADM. @@ -32,6 +31,7 @@ This is the standard workflow for installing PEADM. 1. [Install Bolt](https://www.puppet.com/docs/bolt/latest/bolt_installing) on a jump host. 2. To create a Bolt project that includes the PEADM module, run:
`mkdir && cd && bolt project init --modules puppetlabs-peadm` 3. In the Bolt project directory, update the `inventory.yaml` file with connection information for the servers you plan to use for hosting PE infrastructure. For example: + ```yaml --- groups: @@ -55,56 +55,55 @@ groups: ### What PEADM affects -* The `peadm::install` plan adds a number of custom original identifier (OID) trusted facts to the certificates of deployed PE infrastructure nodes. These trusted facts are used by PEADM plans to identify nodes that host PE infrastructure components. -* Depending on the scale of your architecture, up to four node groups may be created to configure `puppet_enterprise` class parameters for the following PE infrastructure components: - * The primary server - * The primary server replica - * PostgreSQL nodes (database servers) - * Compilers (compiler hosts are designated as belonging to availability group A or B) +- The `peadm::install` plan adds a number of custom original identifier (OID) trusted facts to the certificates of deployed PE infrastructure nodes. These trusted facts are used by PEADM plans to identify nodes that host PE infrastructure components. +- Depending on the scale of your architecture, up to four node groups may be created to configure `puppet_enterprise` class parameters for the following PE infrastructure components: + - The primary server + - The primary server replica + - PostgreSQL nodes (database servers) + - Compilers (compiler hosts are designated as belonging to availability group A or B) ### What PEADM does not affect -* PEADM does not impact regular PE operations. After using it to deploy a new PE installation or upgrade an existing one, PEADM is not required until you want to use it to upgrade PE or expand your installation. -* Using PEADM to install PE or upgrade PE does not prevent you from using documented PE procedures such as setting up disaster recovery or performing a manual upgrade. +- PEADM does not impact regular PE operations. After using it to deploy a new PE installation or upgrade an existing one, PEADM is not required until you want to use it to upgrade PE or expand your installation. +- Using PEADM to install PE or upgrade PE does not prevent you from using documented PE procedures such as setting up disaster recovery or performing a manual upgrade. ### Requirements -* PEADM is compatible with Puppet Enterprise 2019.8.1 or newer versions. -* PEADM requires [Bolt](https://www.puppet.com/docs/bolt/latest/bolt_installing) version 3.17.0 or newer and works on operating systems supported by Bolt. -**Note:** You can use PEADM to install Puppet Enterprise (PE) onto any operating system platform that is supported by the specific PE version you want to install. To review operating system platforms supported by specific PE versions, refer to the PE documentation. -* To successfully convert your current PE installation to a PEADM-managed installation, ensure that the PE setting for editing classifier configuration data is enabled. This setting is enabled by default on new PE installations, but it could be disabled if the relevant configuration was removed from your global hiera.yaml file. See the [PE docs](https://www.puppet.com/docs/pe/latest/config_console.html#enable_console_configuration_data) for more information. +- PEADM is compatible with Puppet Enterprise 2019.8.1 or newer versions. +- PEADM requires [Bolt](https://www.puppet.com/docs/bolt/latest/bolt_installing) version 3.17.0 or newer and works on operating systems supported by Bolt. + **Note:** You can use PEADM to install Puppet Enterprise (PE) onto any operating system platform that is supported by the specific PE version you want to install. To review operating system platforms supported by specific PE versions, refer to the PE documentation. +- To successfully convert your current PE installation to a PEADM-managed installation, ensure that the PE setting for editing classifier configuration data is enabled. This setting is enabled by default on new PE installations, but it could be disabled if the relevant configuration was removed from your global hiera.yaml file. See the [PE docs](https://www.puppet.com/docs/pe/latest/config_console.html#enable_console_configuration_data) for more information. ## Usage For instructions on using PEADM plans, see the following PEADM docs: -* [Install](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/install.md) -* [Upgrade](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/upgrade.md) -* [Convert](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/convert.md) -* [Status](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/status.md) +- [Install](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/install.md) +- [Upgrade](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/upgrade.md) +- [Convert](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/convert.md) +- [Status](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/status.md) ## Reference To understand which architecture is right for you, see the following information on the Puppet documentation site: -* [PE architectures](https://puppet.com/docs/pe/latest/choosing_an_architecture.html) -* [PE multi-region reference architectures](https://puppet.com/docs/patterns-and-tactics/latest/reference-architectures/pe-multi-region-reference-architectures.html) - +- [PE architectures](https://puppet.com/docs/pe/latest/choosing_an_architecture.html) +- [PE multi-region reference architectures](https://puppet.com/docs/patterns-and-tactics/latest/reference-architectures/pe-multi-region-reference-architectures.html) To learn more about the PEADM module and its uses, see the following PEADM docs: -* [Recovery procedures](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/recovery.md) -* [Architectures](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/architectures.md) -* [Expanding deployment](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/expanding.md) -* [Classification](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/classification.md) -* [Testing](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/pre_post_checks.md) -* [Docker based examples](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/docker_examples.md) -* [Release process](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/release_process.md) +- [Recovery procedures](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/recovery.md) +- [Architectures](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/architectures.md) +- [Expanding deployment](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/expanding.md) +- [Classification](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/classification.md) +- [Testing](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/pre_post_checks.md) +- [Docker based examples](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/docker_examples.md) +- [Release process](https://github.com/puppetlabs/puppetlabs-peadm/blob/main/documentation/release_process.md) ## Getting help -* If you find a bug, you can [create a GitHub issue](https://github.com/puppetlabs/puppetlabs-peadm/issues). -* For PE customers using PEADM and experiencing outages or other issues, [contact the Support team](https://portal.perforce.com/s/topic/0TO4X000000DbNgWAK/puppet). +- If you find a bug, you can [create a GitHub issue](https://github.com/puppetlabs/puppetlabs-peadm/issues). +- For PE customers using PEADM and experiencing outages or other issues, [contact the Support team](https://portal.perforce.com/s/topic/0TO4X000000DbNgWAK/puppet). ## License diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 3836e750..db4617fc 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -31,6 +31,10 @@ ) if $arch['disaster-recovery'] { + # Gather certificate extension information from all systems + $cert_extensions = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| { + $memo + { $result.target.peadm::certname => $result['extensions'] } + } $legacy_compiler_a_targets = $legacy_compiler_targets.filter |$index,$target| { $exts = $cert_extensions[$target.peadm::certname()] if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { From 75331662db0ef786ebe1a0ef996d7c1f621589c3 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 16 Sep 2024 16:49:24 +0100 Subject: [PATCH 31/37] feat(convert_compiler_to_legacy): gather certificate extension information - Added steps to gather certificate extension information from all systems using the `peadm::cert_data` task. - Updated variable names for clarity and consistency. - Filtered legacy compiler targets based on availability group using the gathered certificate data. This enhancement improves the accuracy of the conversion process by ensuring that the correct certificate data is used. --- plans/convert_compiler_to_legacy.pp | 66 +++++++++++++++++------------ 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index db4617fc..0486ebc1 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -5,7 +5,7 @@ Optional[Boolean] $remove_pdb = true, ) { $primary_target = peadm::get_targets($primary_host, 1) - $legacy_compiler_targets = peadm::get_targets($legacy_hosts) + $convert_legacy_compiler_targets = peadm::get_targets($legacy_hosts) $cluster = run_task('peadm::get_peadm_config', $primary_host).first.value $error = getvar('cluster.error') @@ -13,21 +13,35 @@ fail_plan($error) } + $replica_host = getvar('cluster.params.replica_host') + $primary_postgresql_host = getvar('cluster.params.primary_postgresql_host') + $replica_postgresql_host = getvar('cluster.params.replica_postgresql_host') + $compiler_hosts = getvar('cluster.params.compiler_hosts') + $legacy_compilers = getvar('cluster.params.legacy_hosts') + + $replica_target = peadm::get_targets($replica_host, 1) + $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) + $compiler_targets = peadm::get_targets($compiler_hosts) + $legacy_targets = peadm::get_targets($legacy_compilers) + $convert_legacy_compiler_targets + $all_targets = peadm::flatten_compact([ - getvar('cluster.params.primary_host'), - getvar('cluster.params.replica_host'), - getvar('cluster.params.primary_postgresql_host'), - getvar('cluster.params.replica_postgresql_host'), - getvar('cluster.params.compiler_hosts'), + $primary_target, + $replica_target, + $primary_postgresql_target, + $replica_postgresql_target, + $compiler_targets, + $legacy_targets, ]) # Ensure input valid for a supported architecture $arch = peadm::assert_supported_architecture( - getvar('cluster.params.primary_host'), - getvar('cluster.params.replica_host'), - getvar('cluster.params.primary_postgresql_host'), - getvar('cluster.params.replica_postgresql_host'), - getvar('cluster.params.compiler_hosts'), + $primary_host, + $replica_host, + $primary_postgresql_host, + $replica_postgresql_host, + $compiler_hosts, + $legacy_compilers, ) if $arch['disaster-recovery'] { @@ -35,7 +49,7 @@ $cert_extensions = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| { $memo + { $result.target.peadm::certname => $result['extensions'] } } - $legacy_compiler_a_targets = $legacy_compiler_targets.filter |$index,$target| { + $legacy_compiler_a_targets = $convert_legacy_compiler_targets.filter |$index,$target| { $exts = $cert_extensions[$target.peadm::certname()] if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { $exts[peadm::oid('peadm_availability_group')] == 'A' @@ -47,7 +61,7 @@ $index % 2 == 0 } } - $legacy_compiler_b_targets = $legacy_compiler_targets.filter |$index,$target| { + $legacy_compiler_b_targets = $convert_legacy_compiler_targets.filter |$index,$target| { $exts = $cert_extensions[$target.peadm::certname()] if ($exts[peadm::oid('peadm_availability_group')] in ['A', 'B']) { $exts[peadm::oid('peadm_availability_group')] == 'B' @@ -60,12 +74,10 @@ } } } else { - $legacy_compiler_a_targets = $legacy_compiler_targets + $legacy_compiler_a_targets = $convert_legacy_compiler_targets $legacy_compiler_b_targets = [] } - $compiler_targets = peadm::get_targets(getvar('cluster.params.compiler_hosts')) - wait([ background('modify-compilers-certs') || { run_plan('peadm::modify_certificate', $compiler_targets, @@ -98,8 +110,8 @@ ]) if $remove_pdb { - run_command('puppet resource service puppet ensure=stopped', $legacy_compiler_targets) - run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $legacy_compiler_targets) + run_command('puppet resource service puppet ensure=stopped', $convert_legacy_compiler_targets) + run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $convert_legacy_compiler_targets) } apply($primary_target) { @@ -121,22 +133,22 @@ } } - run_task('peadm::puppet_runonce', $legacy_compiler_targets) + run_task('peadm::puppet_runonce', $convert_legacy_compiler_targets) run_task('peadm::puppet_runonce', $compiler_targets) run_task('peadm::puppet_runonce', $primary_target) run_task('peadm::puppet_runonce', $all_targets) if $remove_pdb { - run_command('puppet resource package pe-puppetdb ensure=purged', $legacy_compiler_targets) - run_command('puppet resource user pe-puppetdb ensure=absent', $legacy_compiler_targets) + run_command('puppet resource package pe-puppetdb ensure=purged', $convert_legacy_compiler_targets) + run_command('puppet resource user pe-puppetdb ensure=absent', $convert_legacy_compiler_targets) - run_command('rm -rf /etc/puppetlabs/puppetdb', $legacy_compiler_targets) - run_command('rm -rf /var/log/puppetlabs/puppetdb', $legacy_compiler_targets) - run_command('rm -rf /opt/puppetlabs/server/data/puppetdb', $legacy_compiler_targets) + run_command('rm -rf /etc/puppetlabs/puppetdb', $convert_legacy_compiler_targets) + run_command('rm -rf /var/log/puppetlabs/puppetdb', $convert_legacy_compiler_targets) + run_command('rm -rf /opt/puppetlabs/server/data/puppetdb', $convert_legacy_compiler_targets) } - run_command('systemctl start pe-puppetserver.service', $legacy_compiler_targets) - run_command('puppet resource service puppet ensure=running', $legacy_compiler_targets) + run_command('systemctl start pe-puppetserver.service', $convert_legacy_compiler_targets) + run_command('puppet resource service puppet ensure=running', $convert_legacy_compiler_targets) - return("Converted host ${legacy_compiler_targets} to legacy compiler.") + return("Converted host ${convert_legacy_compiler_targets} to legacy compiler.") } From e67d0b34bea880cd8859c44604f40e4b92f86b55 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Tue, 17 Sep 2024 10:16:22 +0100 Subject: [PATCH 32/37] feat(convert_compiler_to_legacy): gather certificate extension information - Updated `puppetdb_host` to include both internal compiler pool addresses. - Modified `convert_compiler_to_legacy` plan to gather certificate extension information from legacy targets using the `peadm::cert_data` task. - Adjusted filtering of legacy compiler targets based on availability group using the gathered certificate data. This enhancement improves the accuracy of the conversion process by ensuring that the correct certificate data is used. --- manifests/setup/legacy_compiler_group.pp | 2 +- plans/convert_compiler_to_legacy.pp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/setup/legacy_compiler_group.pp b/manifests/setup/legacy_compiler_group.pp index 870297ad..e3601200 100644 --- a/manifests/setup/legacy_compiler_group.pp +++ b/manifests/setup/legacy_compiler_group.pp @@ -16,7 +16,7 @@ ], classes => { 'puppet_enterprise::profile::master' => { - 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_a_pool_address].filter |$_| { $_ }, + 'puppetdb_host' => [$internal_compiler_a_pool_address, $internal_compiler_b_pool_address].filter |$_| { $_ }, 'puppetdb_port' => [8081], }, }, diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 0486ebc1..856cbe1c 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -22,7 +22,7 @@ $replica_target = peadm::get_targets($replica_host, 1) $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) - $compiler_targets = peadm::get_targets($compiler_hosts) + $compiler_targets = peadm::get_targets($compiler_hosts) - $convert_legacy_compiler_targets $legacy_targets = peadm::get_targets($legacy_compilers) + $convert_legacy_compiler_targets $all_targets = peadm::flatten_compact([ @@ -46,7 +46,7 @@ if $arch['disaster-recovery'] { # Gather certificate extension information from all systems - $cert_extensions = run_task('peadm::cert_data', $all_targets).reduce({}) |$memo,$result| { + $cert_extensions = run_task('peadm::cert_data', $legacy_targets).reduce({}) |$memo,$result| { $memo + { $result.target.peadm::certname => $result['extensions'] } } $legacy_compiler_a_targets = $convert_legacy_compiler_targets.filter |$index,$target| { From fdac8665fe064d83014b6747c3f86cbf59aca90a Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Tue, 17 Sep 2024 11:29:08 +0100 Subject: [PATCH 33/37] feat(convert_compiler_to_legacy): gather certificate extension information - Added steps to apply `peadm::setup::node_manager_yaml` class on the primary target. - Ensured `primary_host` is set correctly based on the primary target's certname. - Improved the setup process by gathering certificate extension information. This enhancement ensures that the correct certificate data is used, improving the accuracy of the conversion process. --- plans/convert_compiler_to_legacy.pp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 856cbe1c..6489e1ba 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -13,6 +13,15 @@ fail_plan($error) } + apply($primary_target) { + class { 'peadm::setup::node_manager_yaml': + primary_host => $primary_target.peadm::certname() ? { + undef => $primary_target, + default => $primary_target.peadm::certname(), + }, + } + } + $replica_host = getvar('cluster.params.replica_host') $primary_postgresql_host = getvar('cluster.params.primary_postgresql_host') $replica_postgresql_host = getvar('cluster.params.replica_postgresql_host') From e2353248f47664b61ac7a4168b23bca15ecfb5ca Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Tue, 17 Sep 2024 14:54:38 +0100 Subject: [PATCH 34/37] feat(convert_compiler_to_legacy): gather certificate extension information - Added steps to apply `peadm::setup::legacy_compiler_group` class with internal compiler pool addresses. - Ensured `primary_host` is set correctly based on the primary target's certname. - Removed redundant application of `peadm::setup::node_manager_yaml` and `peadm::setup::legacy_compiler_group` classes. This enhancement ensures that the correct certificate data is used, improving the accuracy of the conversion process. --- plans/convert_compiler_to_legacy.pp | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 6489e1ba..1ef29e8e 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -20,6 +20,16 @@ default => $primary_target.peadm::certname(), }, } + + class { 'peadm::setup::legacy_compiler_group': + primary_host => $primary_target.peadm::certname() ? { + undef => $primary_target, + default => $primary_target.peadm::certname(), + }, + internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], + internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], + require => Class['peadm::setup::node_manager_yaml'], + } } $replica_host = getvar('cluster.params.replica_host') @@ -123,25 +133,6 @@ run_command('puppet resource service pe-puppetdb ensure=stopped enable=false', $convert_legacy_compiler_targets) } - apply($primary_target) { - class { 'peadm::setup::node_manager_yaml': - primary_host => $primary_target.peadm::certname() ? { - undef => $primary_target, - default => $primary_target.peadm::certname(), - }, - } - - class { 'peadm::setup::legacy_compiler_group': - primary_host => $primary_target.peadm::certname() ? { - undef => $primary_target, - default => $primary_target.peadm::certname(), - }, - internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], - internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], - require => Class['peadm::setup::node_manager_yaml'], - } - } - run_task('peadm::puppet_runonce', $convert_legacy_compiler_targets) run_task('peadm::puppet_runonce', $compiler_targets) run_task('peadm::puppet_runonce', $primary_target) From 002167c86391aa2987c415c96f71f81b68b903ab Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Tue, 17 Sep 2024 15:58:36 +0100 Subject: [PATCH 35/37] feat(convert_compiler_to_legacy): update internal compiler pool address retrieval - Changed retrieval of `internal_compiler_a_pool_address` and `internal_compiler_b_pool_address` to use `getvar` function. - Ensured compatibility with updated cluster parameter structure. This update improves the robustness of the plan by ensuring correct retrieval of internal compiler pool addresses. --- plans/convert_compiler_to_legacy.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plans/convert_compiler_to_legacy.pp b/plans/convert_compiler_to_legacy.pp index 1ef29e8e..c75924bd 100644 --- a/plans/convert_compiler_to_legacy.pp +++ b/plans/convert_compiler_to_legacy.pp @@ -26,8 +26,8 @@ undef => $primary_target, default => $primary_target.peadm::certname(), }, - internal_compiler_a_pool_address => $cluster['params']['internal_compiler_a_pool_address'], - internal_compiler_b_pool_address => $cluster['params']['internal_compiler_b_pool_address'], + internal_compiler_a_pool_address => getvar('cluster.params.internal_compiler_a_pool_address'), + internal_compiler_b_pool_address => getvar('cluster.params.internal_compiler_b_pool_address'), require => Class['peadm::setup::node_manager_yaml'], } } From 438ca0118aad383fbee695870461c0398e9bc120 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Wed, 18 Sep 2024 10:43:31 +0100 Subject: [PATCH 36/37] ci(github-actions): remove test-upgrade-legacy workflow - Deleted `.github/workflows/test-upgrade-legacy.yaml` file. - Removed workflow for testing PE upgrade with one legacy compiler. This change simplifies the CI configuration by removing an outdated or unnecessary workflow. --- .github/workflows/test-upgrade-legacy.yaml | 151 --------------------- 1 file changed, 151 deletions(-) delete mode 100644 .github/workflows/test-upgrade-legacy.yaml diff --git a/.github/workflows/test-upgrade-legacy.yaml b/.github/workflows/test-upgrade-legacy.yaml deleted file mode 100644 index 2edcd265..00000000 --- a/.github/workflows/test-upgrade-legacy.yaml +++ /dev/null @@ -1,151 +0,0 @@ ---- -name: Upgrade PE with one legacy compiler (test) -on: - pull_request: - paths: - - .github/workflows/**/* - - spec/**/* - - lib/**/* - - tasks/**/* - - functions/**/* - - types/**/* - - plans/**/* - - hiera/**/* - - manifests/**/* - - templates/**/* - - files/**/* - - metadata.json - - Rakefile - - Gemfile - - provision.yaml - - .rspec - - .rubocop.yml - - .puppet-lint.rc - - .fixtures.yml - branches: [main] - workflow_dispatch: - inputs: - ssh-debugging: - description: Boolean; whether or not to pause for ssh debugging - required: true - default: 'false' -jobs: - test-install: - name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }} - runs-on: ubuntu-20.04 - env: - BOLT_GEM: true - BOLT_DISABLE_ANALYTICS: true - LANG: en_US.UTF-8 - strategy: - fail-fast: false - matrix: - architecture: [large-with-dr] - image: [almalinux-cloud/almalinux-8] - version: [2023.7.0] - to_version: [2023.8.0] - steps: - - name: Start SSH session - if: ${{ github.event.inputs.ssh-debugging == 'true' }} - uses: luchihoratiu/debug-via-ssh@main - with: - NGROK_AUTH_TOKEN: ${{ secrets.NGROK_AUTH_TOKEN }} - SSH_PASS: ${{ secrets.SSH_PASS }} - - name: Checkout Source - uses: actions/checkout@v2 - - name: Activate Ruby 2.7 - uses: ruby/setup-ruby@v1 - with: - ruby-version: '2.7' - bundler-cache: true - - name: Print bundle environment - if: ${{ github.repository_owner == 'puppetlabs' }} - run: | - echo ::group::info:bundler - bundle env - echo ::endgroup:: - - name: Provision test cluster - timeout-minutes: 15 - run: | - echo ::group::prepare - mkdir -p $HOME/.ssh - echo 'Host *' > $HOME/.ssh/config - echo ' ServerAliveInterval 150' >> $HOME/.ssh/config - echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config - bundle exec rake spec_prep - echo ::endgroup:: - echo ::group::provision - bundle exec bolt plan run peadm_spec::provision_test_cluster \ - --modulepath spec/fixtures/modules \ - provider=provision_service \ - image=${{ matrix.image }} \ - architecture=${{ matrix.architecture }} - echo ::endgroup:: - echo ::group::info:request - cat request.json || true; echo - echo ::endgroup:: - echo ::group::info:inventory - sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true - echo ::endgroup:: - - name: Set up yq - uses: frenck/action-setup-yq@v1 - with: - version: v4.30.5 - - name: Install PE on test cluster - timeout-minutes: 120 - run: | - bundle exec bolt plan run peadm_spec::install_test_cluster \ - --inventoryfile spec/fixtures/litmus_inventory.yaml \ - --modulepath spec/fixtures/modules \ - architecture=${{ matrix.architecture }} \ - console_password=${{ secrets.CONSOLE_PASSWORD }} \ - version=${{ matrix.version }} - - name: Wait as long as the file ${HOME}/pause file is present - if: ${{ always() && github.event.inputs.ssh-debugging == 'true' }} - run: | - while [ -f "${HOME}/pause" ] ; do - echo "${HOME}/pause present, sleeping for 60 seconds..." - sleep 60 - done - echo "${HOME}/pause absent, continuing workflow." - - name: Convert one compiler to legacy - timeout-minutes: 120 - run: | - echo ::group::prepare - mkdir -p $HOME/.ssh - echo 'Host *' > $HOME/.ssh/config - echo ' ServerAliveInterval 150' >> $HOME/.ssh/config - echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config - bundle exec rake spec_prep - echo ::endgroup:: - primary=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .uri' spec/fixtures/litmus_inventory.yaml) - compiler=$(yq '.groups[].targets[] | select(.vars.role == "compiler") | .uri' spec/fixtures/litmus_inventory.yaml | head -n 1) - echo ::group::convert_compiler_to_legacy - bundle exec bolt plan run peadm::convert_compiler_to_legacy \ - --inventoryfile spec/fixtures/litmus_inventory.yaml \ - --modulepath spec/fixtures/modules \ - --no-host-key-check \ - primary_host=$primary \ - legacy_hosts=$compiler - echo ::endgroup:: - - name: Upgrade PE on test cluster - timeout-minutes: 120 - run: | - bundle exec bolt plan run peadm_spec::upgrade_test_cluster \ - --inventoryfile spec/fixtures/litmus_inventory.yaml \ - --modulepath spec/fixtures/modules \ - --no-host-key-check \ - architecture=${{ matrix.architecture }} \ - version=${{ matrix.to_version }} - - name: Tear down test cluster - if: ${{ always() }} - continue-on-error: true - run: |- - if [ -f spec/fixtures/litmus_inventory.yaml ]; then - echo ::group::tear_down - bundle exec rake 'litmus:tear_down' - echo ::endgroup:: - echo ::group::info:request - cat request.json || true; echo - echo ::endgroup:: - fi From 4a07155fbc93b020199c2bdb53d9ea8c50ce1db9 Mon Sep 17 00:00:00 2001 From: Ioannis Karasavvaidis Date: Mon, 23 Sep 2024 16:58:58 +0100 Subject: [PATCH 37/37] docs: update documentation for converting and upgrading legacy compilers - Updated the version reference from 3.21 to 3.25 in the upgrade documentation. - Added instructions for using the `peadm::convert_compiler_to_legacy` plan. - Removed outdated steps for updating certificate extensions and converting legacy compilers. - Simplified the upgrade process description for Puppet Enterprise using puppetlabs-peadm module 3.25 or later. --- documentation/convert_compiler_to_legacy.md | 9 ++++++ .../upgrade_with_legacy_compilers.md | 28 +++---------------- 2 files changed, 13 insertions(+), 24 deletions(-) create mode 100644 documentation/convert_compiler_to_legacy.md diff --git a/documentation/convert_compiler_to_legacy.md b/documentation/convert_compiler_to_legacy.md new file mode 100644 index 00000000..badb4832 --- /dev/null +++ b/documentation/convert_compiler_to_legacy.md @@ -0,0 +1,9 @@ +# Convert compilers to legacy + +### Puppet Enterprise installed with puppetlabs-peadm version 3.25 or later + +To convert compilers to legacy compilers use the `peadm::convert_compiler_to_legacy` plan. This plan will create the needed Node group and Classifier rules to make compilers legacy. Also will add certificate extensions to those nodes. + +```shell +bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com +``` diff --git a/documentation/upgrade_with_legacy_compilers.md b/documentation/upgrade_with_legacy_compilers.md index 728253ec..398a7bec 100644 --- a/documentation/upgrade_with_legacy_compilers.md +++ b/documentation/upgrade_with_legacy_compilers.md @@ -6,11 +6,11 @@ As a legacy compiler we refer to a compiler that doesn't have PuppetDB. And a cu ## Who is this documentation for -For those users that have installed Puppet Enterprise with puppetlabs-peadm prior version 3.21 and manually converted their existing complilers (all of the or at least 1) to legacy compilers. +For those users that have installed Puppet Enterprise with puppetlabs-peadm prior version 3.25 and manually converted their existing complilers (all of the or at least 1) to legacy compilers. ## Who is this documentation not for -For those users that have installed Puppet Enterprise with PEADM with 3.21 version or later, there is no need to follow this documentation. The install process will automatically have created the necessary configurations for you and you can use the `peadm::convert_compiler_to_legacy` plan if you need a legacy compiler. example: +For those users that have installed Puppet Enterprise with PEADM with 3.25 version or later, there is no need to follow this documentation. The install process will automatically have created the necessary configurations for you and you can use the `peadm::convert_compiler_to_legacy` plan if you need a legacy compiler. example: ```shell bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com @@ -22,26 +22,6 @@ bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.c Usually users pin the nodes in the Pe Master Node Group and then manually removing PuppetDB from compilers nodes. To revert this changes go to your Puppet Enterprise console and unpin the compilers nodes from the Group. -### 2. Update certificate extensions for NON legacy compilers +### 2. Upgrade Puppet Enterprise -If you have NON legacy compilers in your infrastructure, you have to add a certificate extension to them that recognizes them as NON legacy compilers. To do this, execute the following plan: - -```shell -bolt plan run peadm::update_compiler_extensions primary_host=primary.example.com compiler_hosts=compiler1.example.com,compiler2.example.com -``` - -### 3. Use the convert legacy compiler plan - -Now that we have unpinned the compilers nodes from the PE Master node group, execute the following plan to convert your needed compilers to legacy compilers: - -```shell -bolt plan run peadm::convert_compiler_to_legacy legacy_hosts=compiler1.example.com,compiler2.example.com primary_host=primary.example.com -``` - -The above will create the needed Node group and Classifier rules to make the compilers legacy. Also will add certificate extensions to those nodes. - -If you want to keep puppetDB service on the converted compilers, you can do so by passing the `remove_pdb` parameter as `false`. - -### 4. Upgrade Puppet Enterprise - -After you have completed the above steps, you can proceed with the upgrade of Puppet Enterprise as usual using the puppetlabs-peadm module. There is no need to do the above ever again. +You can proceed with the upgrade of Puppet Enterprise as usual using the puppetlabs-peadm module 3.25 or later and pass legacy compilers to the upgrade plan.