Merge remote-tracking branch 'origin/v3-api-breaking-changes' into PM…

…M-8019-fix-bools-in-api

.github/workflows/update.yml

@@ -22,6 +22,7 @@ on:
 jobs:
   build:
     name: Build
+    if: false
 
     strategy:
       fail-fast: false

README.md

@@ -57,7 +57,7 @@ $ docker volume create pmm-data
 3. Run PMM server container
 ```bash
 $ docker run --detach --restart always \
---publish 443:443 \
+--publish 443:8443 \
 --volume pmm-data:/srv \
 --name pmm-server \
 percona/pmm-server:3
@@ -99,4 +99,4 @@ As a general rule of thumb, please try to create bug reports that are:
 
 ## Licensing
 
-Percona is dedicated to **keeping open source open**. Wherever possible, we strive to include permissive licensing for both our software and documentation. For this project, we are using the [GNU AGPLv3](https://github.com/percona/pmm/blob/main/LICENSE) license.
+Percona is dedicated to **keeping open source open**. Wherever possible, we strive to include permissive licensing for both our software and documentation. For this project, we are using the [GNU AGPLv3](./LICENSE) license.

admin/commands/pmm/server/docker/install.go

@@ -129,8 +129,8 @@ func (c *InstallCommand) runContainer(ctx context.Context, volume *volume.Volume
 	logrus.Info("Starting PMM Server")
 
 	ports := nat.PortMap{
-		"443/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: strconv.Itoa(int(c.HTTPSListenPort))}},
-		"80/tcp":  []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: strconv.Itoa(int(c.HTTPListenPort))}},
+		"8443/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: strconv.Itoa(int(c.HTTPSListenPort))}},
+		"8080/tcp": []nat.PortBinding{{HostIP: "0.0.0.0", HostPort: strconv.Itoa(int(c.HTTPListenPort))}},
 	}
 
 	containerID, err := startPMMServer(ctx, volume, "", dockerImage, c.dockerFn, ports, c.ContainerName)

agent/Makefile

@@ -11,7 +11,7 @@ PMM_RELEASE_VERSION ?= $(shell git describe --always --dirty | cut -b2-)
 PMM_RELEASE_TIMESTAMP ?= $(shell date '+%s')
 PMM_RELEASE_FULLCOMMIT ?= $(shell git rev-parse HEAD)
 PMM_RELEASE_BRANCH ?= $(shell git describe --always --contains --all)
-PMM_DEV_SERVER_PORT ?= 443
+PMM_DEV_SERVER_PORT ?= 8443
 ifeq ($(GOBIN),)
 	GOBIN := $(shell go env GOPATH)/bin
 endif

agent/agents/supervisor/supervisor_test.go

@@ -52,7 +52,7 @@ func TestSupervisor(t *testing.T) {
 	cfgStorage := config.NewStorage(&config.Config{
 		Paths:         config.Paths{TempDir: tempDir},
 		Ports:         config.Ports{Min: 65000, Max: 65099},
-		Server:        config.Server{Address: "localhost:443"},
+		Server:        config.Server{Address: "localhost:8443"},
 		LogLinesCount: 1,
 	})
 	s := NewSupervisor(ctx, nil, cfgStorage)

agent/docker-compose.yml

@@ -6,8 +6,8 @@ services:
     image: ${PMM_SERVER_IMAGE:-perconalab/pmm-server:3-dev-latest}
     container_name: pmm-agent_pmm-server
     ports:
-      - "127.0.0.1:80:80"
-      - "127.0.0.1:443:443"
+      - "127.0.0.1:80:8080"
+      - "127.0.0.1:443:8443"
     environment:
       - PMM_DEBUG=1
       - PERCONA_TEST_CHECKS_INTERVAL=10s

api-tests/docker-compose.yml

@@ -6,8 +6,8 @@ services:
     image: ${PMM_SERVER_IMAGE:-perconalab/pmm-server:3-dev-latest}
     container_name: pmm-agent_pmm-server
     ports:
-      - 127.0.0.1:80:80
-      - 127.0.0.1:443:443
+      - 127.0.0.1:80:8080
+      - 127.0.0.1:443:8443
     environment:
       - PMM_DEBUG=1
       - PERCONA_TEST_CHECKS_INTERVAL=10s

api-tests/server/settings_test.go

@@ -507,7 +507,7 @@ func TestSettings(t *testing.T) {
 				assert.Empty(t, res)
 			})
 
-			t.Run("NoAdminUserForSSH", func(t *testing.T) {
+			t.Run("ChangeSSHKey", func(t *testing.T) {
 				defer restoreSettingsDefaults(t)
 
 				sshKey := "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQClY/8sz3w03vA2bY6mBFgUzrvb2FIoHw8ZjUXGGClJzJg5HC" +
@@ -521,7 +521,7 @@ func TestSettings(t *testing.T) {
 					},
 					Context: pmmapitests.Context,
 				})
-				pmmapitests.AssertAPIErrorf(t, err, 500, codes.Internal, `Internal server error.`)
+				require.NoError(t, err)
 				assert.Empty(t, res)
 			})
 

api-tests/server/version_test.go

@@ -34,7 +34,6 @@ import (
 func TestVersion(t *testing.T) {
 	t.Parallel()
 	paths := []string{
-		"managed/v1/version",
 		"v1/version",
 	}
 	for _, path := range paths {

api/nginx/nginx.conf

@@ -3,7 +3,7 @@
 
 daemon off;
 
-error_log stderr info;
+error_log /dev/stderr info;
 # error_log stderr debug;
 
 events {

build/Makefile

@@ -66,36 +66,39 @@ pmm-ami:
 						-var 'pmm_client_repo_name=percona-experimental-x86_64' \
 						-var 'pmm_server_repo=experimental' \
 						-only amazon-ebs -color=false \
-                                                packer/pmm.json
+						packer/pmm.json
 
 pmm-ami-rc:
 	docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build hashicorp/packer:${PACKER_VERSION} \
 			build -var 'pmm_client_repos=original testing' \
-				  	-var 'pmm_client_repo_name=percona-testing-x86_64' \
-					-var 'pmm_server_repo=testing' \
-				  	-only amazon-ebs '-color=false' \
-                                        packer/pmm.json
+						-var 'pmm_client_repo_name=percona-testing-x86_64' \
+						-var 'pmm_server_repo=testing' \
+						-only amazon-ebs '-color=false' \
+						packer/pmm.json
 
 pmm-ami-el9:
 	mkdir -p update && \
-	cp -r ../update/ansible/playbook/* update/ && \
-		sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
+		sed -i 's|become_method: su|become_method: sudo|g' ./roles/postgres/tasks/main.yml && \
 		docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build hashicorp/packer:${PACKER_VERSION} \
 		build -var 'pmm_client_repos=original experimental' \
 					-var 'pmm_client_repo_name=percona-experimental-x86_64' \
 					-var 'pmm_server_repo=experimental' \
 					-only amazon-ebs -color=false \
-                                        packer/pmm.el9.json
+					packer/pmm.el9.json
 
 pmm-ami-el9-rc:
 	mkdir -p update && \
-			cp -r ../update/ansible/playbook/* update/ && \
-			sed -i 's|become_method: su|become_method: sudo|g' update/tasks/roles/postgres/tasks/main.yml && \
-			docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build hashicorp/packer:${PACKER_VERSION} \
-			build -var 'pmm_client_repos=original testing' \
-				  	-var 'pmm_client_repo_name=percona-testing-x86_64' \
+		sed -i 's|become_method: su|become_method: sudo|g' ./roles/postgres/tasks/main.yml && \
+		docker run --rm -v ${HOME}/.aws:/root/.aws -v `pwd`:/build -w /build hashicorp/packer:${PACKER_VERSION} \
+		build -var 'pmm_client_repos=original testing' \
+					-var 'pmm_client_repo_name=percona-testing-x86_64' \
 					-var 'pmm_server_repo=testing' \
-				  	-only amazon-ebs '-color=false' \
-                                        packer/pmm.el9.json
+					-only amazon-ebs '-color=false' \
+					packer/pmm.el9.json
 
 ## ----------------- PACKER ------------------
+
+check:                          ## Run required checkers and linters
+	ansible-playbook --syntax-check ansible/pmm-docker/update.yml
+	ansible-playbook --check ansible/pmm-docker/update.yml
+	ansible-lint ansible/pmm-docker/update.yml

build/ansible/ansible.cfg

@@ -0,0 +1,11 @@
+# This is the default ansible.cfg file.
+# It necessary for ansible to work properly when it acts as 'pmm' user.
+# Otherwise, it will fail with 'Permission denied' error since the default paths are '/root/.ansible/tmp'
+# Ref: https://github.com/ansible/ansible/blob/stable-2.9/examples/ansible.cfg
+[defaults]
+
+remote_tmp     = /tmp
+local_tmp      = /tmp
+
+# additional paths to search for roles in, colon separated
+roles_path     = /opt/ansible/roles

update/ansible/playbook/tasks/init.yml → build/ansible/pmm-docker/init.yml

@@ -2,8 +2,10 @@
 # This playbook contains tasks executed during initialization PMM Server
 - hosts: localhost
   become: true
+  become_method: su
+  become_user: pmm
   gather_facts: true
-  tasks:
-    - name: Run initialization role
-      include_role:
-        name: initialization
+
+
+  roles:
+    - initialization

build/ansible/pmm-docker/update.yml

@@ -0,0 +1,196 @@
+---
+# This playbook contains tasks executed during PMM Server update.
+- hosts: localhost
+  become: true
+  remote_user: root
+  gather_facts: true
+
+  environment:
+    PATH: /usr/local/bin:{{ ansible_env.PATH }}
+
+  pre_tasks:
+    - name: Detect /srv/pmm-distribution
+      stat:
+        path: /srv/pmm-distribution
+      no_log: true
+      register: srv_pmm_distribution
+
+    - name: Detect container environment
+      set_fact:
+        is_docker: '{{ lookup("file", "/srv/pmm-distribution") == "docker" }}'
+      no_log: true
+      when: srv_pmm_distribution.stat.exists
+
+    - name: Set the variable to true if undefined
+      set_fact:
+        is_docker: true
+      when: is_docker is undefined
+
+  tasks:
+    - name: Enable maintenance mode
+      copy:
+        src: maintenance.html
+        dest: /usr/share/pmm-server/maintenance/
+        owner: pmm
+        group: pmm
+        mode: 0644
+
+    # restart pmm-managed-init and pmm-managed first as they may update supervisord configuration on start
+    - name: Generate new supervisor config
+      command: pmm-managed-init
+      register: managed_init_result
+      changed_when: True
+
+    - name: Disable pmm-update-perform-init
+      ini_file:
+        path: /etc/supervisord.d/pmm.ini
+        section: program:pmm-update-perform-init
+        option: autostart
+        value: "false"
+
+    - name: Check that supervisor socket exists
+      stat:
+        path: /run/supervisor/supervisor.sock
+      register: supervisor_socket
+
+    # During build time, this will be the first start of supervisord.
+    - name: Start supervisord
+      when: not supervisor_socket.stat.exists
+      shell: supervisord -c /etc/supervisord.conf &
+
+    - name: Wait until postgres port is present
+      wait_for:
+        host: localhost
+        port: 5432
+        timeout: 150
+
+    - name: Run initialization playbook
+      include_role:
+        name: initialization
+
+    # See https://github.com/Supervisor/supervisor/issues/1264 for explanation
+    # why we do reread + stop/remove/add instead of using supervisorctl Ansible module.
+    - name: Reread supervisord configuration
+      command: supervisorctl reread
+      become: true
+      become_user: pmm
+      become_method: su
+      register: reread_result
+      changed_when: "'No config updates to processes' not in reread_result.stdout"
+
+    - name: Check reread results
+      debug: var=reread_result.stdout_lines
+
+    - name: Restart pmm-managed
+      command: "supervisorctl {{ item }} pmm-managed"
+      become: true
+      become_user: pmm
+      become_method: su
+      loop: 
+        - stop
+        - remove
+        - add
+
+    # Give pmm-managed time to update supervisord configuration,
+    # and give update UI time to catch up after pmm-managed restart
+    - name: Wait for pmm-managed
+      pause: seconds=10
+
+    # Fix things that should be fixed before restarts.
+
+    - name: Reread supervisord configuration again
+      command: supervisorctl reread
+      register: reread_result
+      changed_when: "'No config updates to processes' not in reread_result.stdout"
+
+    - name: Check reread results
+      debug: var=reread_result.stdout_lines
+
+    - name: Restart services
+      command: supervisorctl {{ item.1 }} {{ item.0 }}
+      become: true
+      become_user: pmm
+      become_method: su
+      # changed_when: true
+      with_nested:
+        - - nginx
+          - grafana
+          - qan-api2
+          - pmm-agent
+        - ["stop", "remove", "add"]
+
+    - name: Check supervisord logs
+      shell: sleep 10 && tail -n 200 /srv/logs/supervisord.log
+
+    - name: Check grafana logs
+      shell: cat /srv/logs/grafana.log
+
+    - name: Fix grafana fields type
+      postgresql_query:
+        db: grafana
+        query: "{{ item }}"
+      loop:
+        - ALTER TABLE tag ALTER COLUMN key TYPE text;
+        - ALTER TABLE tag ALTER COLUMN value TYPE text;
+        - ALTER TABLE api_key ALTER COLUMN key TYPE text;
+        - ALTER TABLE api_key ALTER COLUMN name TYPE text;
+      when: not ansible_check_mode
+
+    - name: Change default admin id
+      postgresql_query:
+        db: grafana
+        query: UPDATE "user" SET id='1' WHERE login='admin';
+      when: not ansible_check_mode
+
+    # - name: Remove redundant packages
+    #   yum:
+    #     state: absent
+    #     name:
+    #       - logrotate # https://jira.percona.com/browse/PMM-7627
+
+    # Regenerating pmm.ini and enabling pmm-update-perform-init
+    - name: Generate new supervisor config
+      command: pmm-managed-init
+      become: true
+      become_user: pmm
+      become_method: su      
+      register: managed_init_result
+      changed_when: True
+
+    - name: Reread pmm-update-perform-init supervisor config
+      command: supervisorctl reread
+      register: reread_init__result
+      changed_when: "'No config updates to processes' not in reread_init__result.stdout"
+
+    - name: Update/restart other services
+      command: supervisorctl update
+      register: update_result
+      changed_when: "'updated' in update_result.stdout"
+
+    - name: Print other services's logs
+      debug: var=update_result.stdout_lines
+
+    - name: Wait for PMM to be ready
+      ansible.builtin.uri:
+        url: "http://127.0.0.1:7772/v1/readyz"
+        status_code: 200
+        method: GET
+      register: healthcheck
+      until: healthcheck is not failed
+      retries: 120
+      delay: 1
+
+    # SIGUSR2 is sent to supervisord by pmm-managed right before the update for logging to work correctly.
+    # We use that fact to show what was restarted during the update.
+    - name: Get supervisord logs
+      shell: supervisorctl maintail -100000 | tac | awk '!flag; /received SIGUSR2/{flag = 1};' | tac
+      register: maintail_result
+      changed_when: False
+
+    - name: Print supervisord logs
+      debug: var=maintail_result.stdout_lines
+
+    - name: Disable maintenance mode
+      file:
+        state: absent
+        path: /usr/share/pmm-server/maintenance/maintenance.html

update/ansible/playbook/tasks/create-lvm.yml → build/ansible/pmm/create-lvm.yml

@@ -1,3 +1,4 @@
+# TODO: This role seems to no longer be used. Verify and remove.
 - hosts: localhost
   become: true
   gather_facts: true