From 101265c532243f3667f35ff2fd5f74d3ff462716 Mon Sep 17 00:00:00 2001 From: Alex Tymchuk Date: Wed, 13 Sep 2023 23:15:40 +0300 Subject: [PATCH 1/3] PMM-12457 add node_name to agent metrics (#2474) * PMM-12457 add node_name to agent metrics * PMM-12457 update the template to use node_name * PMM-12457 fix punctuation --- managed/data/iatemplates/agent_down.yml | 5 ++--- .../services/inventory/inventory_metrics.go | 18 +++++++++++++++--- .../inventory/inventory_metrics_test.go | 8 ++++---- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/managed/data/iatemplates/agent_down.yml b/managed/data/iatemplates/agent_down.yml index e913696f6eb..89abeb7a124 100644 --- a/managed/data/iatemplates/agent_down.yml +++ b/managed/data/iatemplates/agent_down.yml @@ -8,6 +8,5 @@ templates: severity: critical annotations: description: |- - PMM agent on node '{{ $labels.node_id }}' cannot be reached. Host may be down. - summary: PMM agent on node '{{ $labels.node_id }}' cannot be reached. Host may be down. - + PMM agent on node '{{ $labels.node_name }}', node ID '{{ $labels.node_id }}', cannot be reached. Host may be down. + summary: PMM agent on node '{{ $labels.node_name }}' cannot be reached. Host may be down. diff --git a/managed/services/inventory/inventory_metrics.go b/managed/services/inventory/inventory_metrics.go index afda82c0b6a..d390b95b733 100644 --- a/managed/services/inventory/inventory_metrics.go +++ b/managed/services/inventory/inventory_metrics.go @@ -68,7 +68,7 @@ func NewInventoryMetricsCollector(metrics inventoryMetrics) *InventoryMetricsCol mAgentsDesc: prom.NewDesc( prom.BuildFQName(prometheusNamespace, prometheusSubsystem, "agents"), "Inventory Agent", - []string{"agent_id", "agent_type", "service_id", "node_id", "pmm_agent_id", "disabled", "version"}, + []string{"agent_id", "agent_type", "service_id", "node_id", "node_name", "pmm_agent_id", "disabled", "version"}, nil), mNodesDesc: prom.NewDesc( prom.BuildFQName(prometheusNamespace, prometheusSubsystem, "nodes"), @@ -85,7 +85,7 @@ func NewInventoryMetricsCollector(metrics inventoryMetrics) *InventoryMetricsCol } } -func GetRunsOnNodeIDByPMMAgentID(agents []*models.Agent, pmmAgentID string) string { +func getRunsOnNodeIDByPMMAgentID(agents []*models.Agent, pmmAgentID string) string { for _, agent := range agents { if agent.AgentID == pmmAgentID { return pointer.GetString(agent.RunsOnNodeID) @@ -103,6 +103,16 @@ func (i *InventoryMetrics) GetAgentMetrics(ctx context.Context) ([]Metric, error return err } + dbNodes, err := models.FindNodes(tx.Querier, models.NodeFilters{}) + if err != nil { + return err + } + + nodeMap := make(map[string]string, len(dbNodes)) + for _, node := range dbNodes { + nodeMap[node.NodeID] = node.NodeName + } + for _, agent := range dbAgents { runsOnNodeID := "" disabled := "0" @@ -121,14 +131,16 @@ func (i *InventoryMetrics) GetAgentMetrics(ctx context.Context) ([]Metric, error runsOnNodeID = pointer.GetString(agent.RunsOnNodeID) } else { metricValue = float64(inventorypb.AgentStatus_value[agent.Status]) - runsOnNodeID = GetRunsOnNodeIDByPMMAgentID(dbAgents, pmmAgentID) + runsOnNodeID = getRunsOnNodeIDByPMMAgentID(dbAgents, pmmAgentID) } + nodeName := nodeMap[runsOnNodeID] agentMetricLabels := []string{ agent.AgentID, string(agent.AgentType), pointer.GetString(agent.ServiceID), runsOnNodeID, + nodeName, pmmAgentID, disabled, pointer.GetString(agent.Version), diff --git a/managed/services/inventory/inventory_metrics_test.go b/managed/services/inventory/inventory_metrics_test.go index c65631588f3..939ccc7b074 100644 --- a/managed/services/inventory/inventory_metrics_test.go +++ b/managed/services/inventory/inventory_metrics_test.go @@ -38,7 +38,7 @@ func TestNewInventoryMetricsCollector(t *testing.T) { agentMetrics := []Metric{ { - labels: []string{"A1", string(models.PMMAgentType), "S1", "N1", "PA1", strconv.Itoa(1), "V1"}, + labels: []string{"A1", string(models.PMMAgentType), "S1", "N1", "NN1", "PA1", strconv.Itoa(1), "V1"}, value: float64(1), }, } @@ -87,13 +87,13 @@ func TestNewInventoryMetricsCollector(t *testing.T) { const expectedAgentMetrics = ` # HELP pmm_managed_inventory_agents Inventory Agent # TYPE pmm_managed_inventory_agents gauge - pmm_managed_inventory_agents{agent_id="A1",agent_type="pmm-agent",disabled="1",node_id="N1",pmm_agent_id="PA1",service_id="S1",version="V1"} 1 + pmm_managed_inventory_agents{agent_id="A1",agent_type="pmm-agent",disabled="1",node_id="N1",node_name="NN1",pmm_agent_id="PA1",service_id="S1",version="V1"} 1 ` const expectedNodeMetrics = ` # HELP pmm_managed_inventory_nodes Inventory Node - # TYPE pmm_managed_inventory_nodes gauge - pmm_managed_inventory_nodes{container_name="C1",node_id="N1",node_name="N1",node_type="generic"} 1 + # TYPE pmm_managed_inventory_nodes gauge + pmm_managed_inventory_nodes{container_name="C1",node_id="N1",node_name="N1",node_type="generic"} 1 ` const expectedServiceMetrics = ` From 1cb518a244480bd94a0ce2ed0782a1be49fe6d47 Mon Sep 17 00:00:00 2001 From: Alex Tymchuk Date: Fri, 15 Sep 2023 10:35:08 +0300 Subject: [PATCH 2/3] PMM-12488 update VictoriaMetrics to v1.93.4 (#2483) --- build/packages/rpm/server/SPECS/pmm-dump.spec | 1 + build/packages/rpm/server/SPECS/victoriametrics.spec | 11 +++++++---- build/scripts/vars | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/build/packages/rpm/server/SPECS/pmm-dump.spec b/build/packages/rpm/server/SPECS/pmm-dump.spec index 24ae2ee76c1..c4175cda2c1 100644 --- a/build/packages/rpm/server/SPECS/pmm-dump.spec +++ b/build/packages/rpm/server/SPECS/pmm-dump.spec @@ -31,6 +31,7 @@ install -d -p %{buildroot}%{_sbindir} install -p -m 0755 pmm-dump %{buildroot}%{_sbindir}/pmm-dump %files +%license LICENSE %doc README.md %{_sbindir}/pmm-dump diff --git a/build/packages/rpm/server/SPECS/victoriametrics.spec b/build/packages/rpm/server/SPECS/victoriametrics.spec index 7b13cd2f3e1..ec55d239d15 100644 --- a/build/packages/rpm/server/SPECS/victoriametrics.spec +++ b/build/packages/rpm/server/SPECS/victoriametrics.spec @@ -9,11 +9,11 @@ %global repo VictoriaMetrics %global provider github.com/VictoriaMetrics/%{repo} -%global commit pmm-6401-v1.93.1 +%global commit pmm-6401-v1.93.4 Name: percona-victoriametrics -Version: 1.93.1 -Release: 2%{?dist} +Version: 1.93.4 +Release: 1%{?dist} Summary: VictoriaMetrics monitoring solution and time series database License: Apache-2.0 URL: https://%{provider} @@ -43,13 +43,16 @@ install -D -p -m 0755 ./bin/vmalert-pure %{buildroot}%{_sbindir}/vmalert %files -%copying LICENSE +%license LICENSE %doc README.md %{_sbindir}/victoriametrics %{_sbindir}/vmalert %changelog +* Thu Sep 14 2023 Alex Tymchuk - 1.93.4-1 +- upgrade victoriametrics to 1.93.4 release + * Fri Sep 1 2023 Nurlan Moldomurov - 1.93.1-2 - upgrade victoriametrics to 1.93.1 release diff --git a/build/scripts/vars b/build/scripts/vars index f1597dd14ab..789377c71b6 100644 --- a/build/scripts/vars +++ b/build/scripts/vars @@ -55,5 +55,5 @@ docker_client_tarball=${root_dir}/results/docker/pmm2-client-${pmm_version}.dock source_tarball=${root_dir}/results/source_tarball/pmm2-client-${pmm_version}.tar.gz binary_tarball=${root_dir}/results/tarball/pmm2-client-${pmm_version}.tar.gz -# https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/pmm-6401-v1.93.1 -vmagent_commit_hash=ca20478a69233179b1891755937b5c7d58791fc2 +# https://github.com/VictoriaMetrics/VictoriaMetrics/releases/tag/pmm-6401-v1.93.4 +vmagent_commit_hash=58ecb9066574f38f1d1c91ace467316e7f175b09 From 576142adcd752462868575a238bd97804b22449a Mon Sep 17 00:00:00 2001 From: Talha Bin Rizwan Date: Fri, 15 Sep 2023 16:35:08 +0500 Subject: [PATCH 3/3] PMM-9769 update all components (#2478) * PMM-9769 Use ansible-core instead of ansible package. * PMM-9769 Update all components. --- build/ansible/roles/ami-ovf/tasks/main.yml | 31 +++++++++++++- build/ansible/roles/cloud-node/tasks/main.yml | 23 ++++++++-- .../ansible/roles/pmm2-images/tasks/main.yml | 15 ++++++- build/docker/server/Dockerfile.el9 | 21 ++++------ build/packer/pmm2.el9.json | 3 +- update/ansible/playbook/tasks/update.yml | 42 ++++++++++++++++++- 6 files changed, 111 insertions(+), 24 deletions(-) diff --git a/build/ansible/roles/ami-ovf/tasks/main.yml b/build/ansible/roles/ami-ovf/tasks/main.yml index 6fdec5edd75..fd30f8009f3 100644 --- a/build/ansible/roles/ami-ovf/tasks/main.yml +++ b/build/ansible/roles/ami-ovf/tasks/main.yml @@ -31,12 +31,39 @@ enabled: yes gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY -- name: Disable SELinux +- name: Disable SELinux | EL7 + when: + - ansible_distribution == 'CentOS' + - ansible_distribution_major_version == '7' selinux: policy: targeted state: permissive -- name: PMM | Add firewalld rule +- name: Disable SELinux | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + vars: + ansible_python_interpreter: /usr/bin/python3.9 + selinux: + policy: targeted + state: permissive + +- name: Add firewalld rule | EL7 + when: + - ansible_distribution == 'CentOS' + - ansible_distribution_major_version == '7' + firewalld: port={{ item }} permanent=true state=enabled immediate=yes + with_items: + - 80/tcp + - 443/tcp + +- name: Add firewalld rule | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + vars: + ansible_python_interpreter: /usr/bin/python3.9 firewalld: port={{ item }} permanent=true state=enabled immediate=yes with_items: - 80/tcp diff --git a/build/ansible/roles/cloud-node/tasks/main.yml b/build/ansible/roles/cloud-node/tasks/main.yml index edcbe3104d7..b5fb6bcf6be 100644 --- a/build/ansible/roles/cloud-node/tasks/main.yml +++ b/build/ansible/roles/cloud-node/tasks/main.yml @@ -38,11 +38,26 @@ - yum-utils - cloud-init - firewalld - - python3-pip - - ansible + - python3-libselinux + - python3-firewall + +- name: Firewalld | Start EL7 + when: + - ansible_distribution == 'CentOS' + - ansible_distribution_major_version == '7' + - ansible_os_family == 'RedHat' + service: + name: firewalld + state: started + enabled: yes -- name: Firewalld | Start - when: ansible_os_family == 'RedHat' +- name: Firewalld | Start EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + - ansible_os_family == 'RedHat' + vars: + ansible_python_interpreter: /usr/bin/python3.9 service: name: firewalld state: started diff --git a/build/ansible/roles/pmm2-images/tasks/main.yml b/build/ansible/roles/pmm2-images/tasks/main.yml index f7812a6328c..6fed510de72 100644 --- a/build/ansible/roles/pmm2-images/tasks/main.yml +++ b/build/ansible/roles/pmm2-images/tasks/main.yml @@ -47,13 +47,25 @@ state: installed ignore_errors: True -- name: Packages | Update OS +- name: Packages | Update OS EL7 + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == "7" yum: name: "*" state: latest exclude: "ansible*" disablerepo: percona-release-x86_64 +- name: Packages | Update OS EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + yum: + name: "*" + state: latest + disablerepo: percona-release-x86_64 + - name: Packages | Install OS tools for EL7 when: - ansible_distribution == "CentOS" @@ -70,7 +82,6 @@ name: - python3-pip - python3.11-pip - - python3.11 - python3.11-psycopg2 - rsync - libsqlite3x-devel # package does not come pre-installed on EL9 diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9 index 925e4e6c08a..4fb9001f98a 100644 --- a/build/docker/server/Dockerfile.el9 +++ b/build/docker/server/Dockerfile.el9 @@ -18,19 +18,14 @@ EXPOSE 80 443 WORKDIR /opt -# NOTE: Ansible should NOT be installed via yum/dnf -# Read more: https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html#pip-install -# RUN microdnf -y install yum && yum -y install python3-pip && \ -# yum -y install oracle-epel-release-el9 ansible-core && \ -# python3 -m pip install ansible && \ -# python3 -m pip install setuptools && \ -# yum -y install epel-release - -RUN microdnf -y install yum && yum -y install python3-pip && \ - yum -y install oracle-epel-release-el9 ansible-core && \ - yum -y install epel-release && \ - yum -y install glibc-langpack-en && \ - yum -y install ansible vi +RUN microdnf -y install epel-release && \ + microdnf -y install ansible-core \ + ansible-collection-community-general \ + ansible-collection-community-postgresql \ + ansible-collection-ansible-posix \ + glibc-langpack-en \ + yum \ + vi COPY RPMS /tmp/RPMS COPY gitCommit /tmp/gitCommit diff --git a/build/packer/pmm2.el9.json b/build/packer/pmm2.el9.json index 51488569db2..baa0ef18f2f 100644 --- a/build/packer/pmm2.el9.json +++ b/build/packer/pmm2.el9.json @@ -114,7 +114,8 @@ "inline": [ "sudo yum -y update", "sudo yum -y install epel-release", - "sudo yum -y install ansible" + "sudo yum -y install ansible-core", + "sudo yum -y install ansible-collection-community-general ansible-collection-community-postgresql ansible-collection-ansible-posix" ] }, { diff --git a/update/ansible/playbook/tasks/update.yml b/update/ansible/playbook/tasks/update.yml index db484e0cae4..ad747de74a9 100644 --- a/update/ansible/playbook/tasks/update.yml +++ b/update/ansible/playbook/tasks/update.yml @@ -239,7 +239,10 @@ - name: Wait for pmm-managed pause: seconds=10 - - name: Update system packages + - name: Update system packages EL7 + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == "7" yum: name: "*" state: latest @@ -247,7 +250,10 @@ exclude: - nginx* - - name: Updating only select packages + - name: Updating only select packages EL7 + when: + - ansible_distribution == "CentOS" + - ansible_distribution_major_version == "7" yum: name: "{{ item }}" state: latest @@ -258,6 +264,38 @@ - sshpass - vi + - name: Remove ansible RPM if installed | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + yum: + name: ansible + state: absent + ignore_errors: yes + + - name: Install ansible-core RPM | EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + yum: + name: + - ansible-core + - ansible-collection-community-general + - ansible-collection-community-postgresql + - ansible-collection-ansible-posix + state: present + + - name: Update system packages EL9 + when: + - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' + - ansible_distribution_major_version == '9' + - not ansible_check_mode + yum: + name: "*" + state: latest + exclude: + - nginx* + - name: Install nginx include_role: name: nginx