From 1c36efe6f8f85d612ccfb5a73c1c913bd50a24d0 Mon Sep 17 00:00:00 2001 From: Talha Bin Rizwan Date: Mon, 26 Aug 2024 12:31:28 +0500 Subject: [PATCH] PMM-13244 Drop amazon linux 2. (#3169) * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * PMM-13244 Drop amazon linux 2. * Update .gitignore Co-authored-by: Alex Demidoff * Update build/ansible/pmm2/post-build-actions.yml Co-authored-by: Alex Demidoff * Update build/ansible/roles/pmm2-images/tasks/main.yml Co-authored-by: Alex Demidoff * Update build/docker/server/Dockerfile.el9 Co-authored-by: Alex Demidoff * PMM-13244 Update build/ansible/roles/supervisord-init/tasks/main.yml * PMM-13244 Use Ansible var instead of Docker env var. * Update build/packer/aws.pkr.hcl Co-authored-by: Alex Demidoff * Update build/packer/aws.pkr.hcl Co-authored-by: Alex Demidoff --------- Co-authored-by: Alex Demidoff --- .gitignore | 2 + build/ansible/pmm2/post-build-actions.yml | 24 ++++---- .../ansible/roles/pmm2-images/tasks/main.yml | 12 ++-- build/ansible/roles/pmm2-images/vars/main.yml | 2 + .../roles/supervisord-init/tasks/main.yml | 6 +- build/docker/server/Dockerfile.el9 | 6 +- build/packer/ansible/agent-aws.yml | 57 ++++++++++++------- build/packer/aws.pkr.hcl | 8 +-- 8 files changed, 69 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 449b577734..f17fdd9d00 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ cli-tests/playwright-report/ api-tests/pmm-api-tests-output.txt api-tests/pmm-api-tests-junit-report.xml + +packer.log diff --git a/build/ansible/pmm2/post-build-actions.yml b/build/ansible/pmm2/post-build-actions.yml index c9b61c24dc..18d160884a 100644 --- a/build/ansible/pmm2/post-build-actions.yml +++ b/build/ansible/pmm2/post-build-actions.yml @@ -7,11 +7,12 @@ vars: pmm_client_repos: "pmm2-client testing" pmm_client_repos_final: "pmm2-client release" + pmm_running_in_docker: false # Set a default value tasks: # pmm-managed checks that if /srv/pmm-distribution exist, it contains "docker", "ovf", or "ami" (all lowercase) - name: Detect distribution | Create '/srv/pmm-distribution' file for Docker - when: ansible_virtualization_type == "docker" + when: pmm_running_in_docker copy: content: "docker" dest: /srv/pmm-distribution @@ -28,6 +29,7 @@ ( ansible_virtualization_type == "xen" or ansible_virtualization_type == "kvm" ) and ansible_system_vendor != "DigitalOcean" + and not pmm_running_in_docker copy: content: "ami" dest: /srv/pmm-distribution @@ -50,14 +52,14 @@ state: present update_cache: yes when: - - ansible_virtualization_type != "docker" + - not pmm_running_in_docker - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - ansible_distribution_major_version == '9' - name: Set locale to en_US.utf8 | EL9 command: localectl set-locale LANG=en_US.utf8 when: - - ansible_virtualization_type != "docker" + - not pmm_running_in_docker - ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux' - ansible_distribution_major_version == '9' @@ -102,11 +104,11 @@ # ignore_errors: True - name: Supervisord stop | Stop supervisord service for AMI/OVF - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker service: name=supervisord state=stopped enabled=yes - name: Supervisord stop | Stop supervisord service for docker - when: ansible_virtualization_type == "docker" + when: pmm_running_in_docker shell: supervisorctl shutdown # PMM-11336 - The previous steps failed to start PostgreSQL using supervisord, @@ -118,14 +120,14 @@ become: yes become_user: postgres ignore_errors: yes - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker - name: Start PostgreSQL database without supervisord command: /usr/pgsql-14/bin/pg_ctl start -D /srv/postgres14 become: yes become_user: postgres ignore_errors: yes - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker - name: Remove pmm-managed database EL7 when: ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' @@ -160,18 +162,18 @@ become: yes become_user: postgres ignore_errors: yes - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker - name: Supervisord stop | Stop supervisord service for AMI/OVF - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker service: name=supervisord state=stopped enabled=yes - name: Supervisord stop EL7 | Stop supervisord service for docker - when: ansible_virtualization_type == "docker" and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' + when: pmm_running_in_docker and ansible_distribution == 'CentOS' and ansible_distribution_major_version == '7' shell: supervisorctl shutdown - name: Supervisord stop EL9 | Stop supervisord service for docker - when: ansible_virtualization_type == "docker" and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' + when: pmm_running_in_docker and (ansible_distribution == 'OracleLinux' or ansible_distribution == 'AlmaLinux') and ansible_distribution_major_version == '9' shell: /usr/local/bin/supervisorctl shutdown - name: Cleanup yum cache | Cleanup yum cache diff --git a/build/ansible/roles/pmm2-images/tasks/main.yml b/build/ansible/roles/pmm2-images/tasks/main.yml index 3ab716ba59..2a7c28ee66 100644 --- a/build/ansible/roles/pmm2-images/tasks/main.yml +++ b/build/ansible/roles/pmm2-images/tasks/main.yml @@ -29,7 +29,7 @@ # local yum repo for building of pmm server docker image in autobuild jobs - name: PMM | Add local YUM repository - when: ansible_virtualization_type == "docker" + when: pmm_running_in_docker yum_repository: name: local description: Local YUM repository - x86_64 @@ -89,7 +89,7 @@ - name: Create users for non-docker images | Create users user: name: "pmm" - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker - name: Create users and groups in docker container block: @@ -120,7 +120,7 @@ - { name: grafana, uid: 998, comment: "Grafana Dashboard", shell: "/sbin/nologin", home: "/etc/grafana", group: grafana, } - { name: clickhouse, uid: 997, comment: "Clickhouse server", shell: "/sbin/nologin", home: "/var/lib/clickhouse", group: clickhouse, } - { name: pmm-agent, uid: 996, comment: "pmm-agent", shell: "/bin/false", home: "/usr/local/percona/", group: pmm-agent, } - when: ansible_virtualization_type == "docker" + when: pmm_running_in_docker - name: Create directories | Create dirs file: path={{ item }} state=directory owner=pmm group=pmm @@ -138,7 +138,7 @@ mode: "0775" - name: Create dirs | Create dirs - when: ansible_virtualization_type == "docker" + when: pmm_running_in_docker file: path={{ item }} state=directory with_items: - /var/lib/cloud/scripts/per-once @@ -178,11 +178,11 @@ enablerepo: "{{ pmm_client_repo_name }}" - name: Disable pmm-agent service | Disable pmm-agent - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker service: name=pmm-agent state=stopped enabled=no - name: Create tmp dirs | Create tmp dirs - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker command: /usr/bin/systemd-tmpfiles --create --remove --boot --exclude-prefix=/dev - name: Copy grafana.ini file for the first run diff --git a/build/ansible/roles/pmm2-images/vars/main.yml b/build/ansible/roles/pmm2-images/vars/main.yml index 7410e4a875..4469094387 100644 --- a/build/ansible/roles/pmm2-images/vars/main.yml +++ b/build/ansible/roles/pmm2-images/vars/main.yml @@ -4,3 +4,5 @@ pmm_client_repos: "pmm2-client testing" # line below is sed'ed by build-server-docker script pmm_client_repo_name: "pmm2-client-testing-x86_64" + +pmm_running_in_docker: false # Set a default value diff --git a/build/ansible/roles/supervisord-init/tasks/main.yml b/build/ansible/roles/supervisord-init/tasks/main.yml index 72d87df658..2001ddd5f4 100644 --- a/build/ansible/roles/supervisord-init/tasks/main.yml +++ b/build/ansible/roles/supervisord-init/tasks/main.yml @@ -69,7 +69,7 @@ state: directory - name: Configure supervisor | Add /etc/tmpfiles.d/supervisor.conf config - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker copy: content: | D /var/run/supervisor 0775 root root - @@ -91,7 +91,7 @@ value: dummy - name: Configure supervisor | Increase number of open files for jobs - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker ini_file: dest: /etc/supervisord.conf section: supervisord @@ -99,7 +99,7 @@ value: "800000" - name: Configure supervisor | Add supervisord.service - when: ansible_virtualization_type != "docker" + when: not pmm_running_in_docker copy: src: supervisord.service dest: /usr/lib/systemd/system/ diff --git a/build/docker/server/Dockerfile.el9 b/build/docker/server/Dockerfile.el9 index 4fb9001f98..b8b295b671 100644 --- a/build/docker/server/Dockerfile.el9 +++ b/build/docker/server/Dockerfile.el9 @@ -33,9 +33,9 @@ COPY gitCommit /tmp/gitCommit COPY ansible /opt/ansible # NOTE: this needs to be refactored, since some of the playbooks are duplicates RUN cp -r /opt/ansible/roles /opt/ansible/pmm2-docker/roles -RUN ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2-docker/main.yml \ - && ansible-playbook -vvv -i 'localhost,' -c local /usr/share/pmm-update/ansible/playbook/tasks/update.yml \ - && ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2/post-build-actions.yml +RUN ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2-docker/main.yml -e "pmm_running_in_docker=true" \ + && ansible-playbook -vvv -i 'localhost,' -c local /usr/share/pmm-update/ansible/playbook/tasks/update.yml -e "pmm_running_in_docker=true" \ + && ansible-playbook -vvv -i 'localhost,' -c local /opt/ansible/pmm2/post-build-actions.yml -e "pmm_running_in_docker=true" COPY entrypoint.sh /opt/entrypoint.sh HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 CMD curl -f http://127.0.0.1/v1/readyz || exit 1 diff --git a/build/packer/ansible/agent-aws.yml b/build/packer/ansible/agent-aws.yml index f5d8c84ef9..63075ed46e 100644 --- a/build/packer/ansible/agent-aws.yml +++ b/build/packer/ansible/agent-aws.yml @@ -22,10 +22,36 @@ - name: Add a repository for NodeJS shell: "curl -fsSL https://rpm.nodesource.com/setup_{{ node_version }} | bash -" + - name: Install Oracle EPEL Repo + yum: + name: oracle-epel-release-el9 + state: latest + + - name: Install yum-utils + yum: + name: yum-utils + state: latest + + - name: Add Docker repository + command: yum-config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo + + - name: Install Docker packages + yum: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: latest + + - name: Enable Docker service + systemd: + name: docker + enabled: yes + state: started + - name: Install packages - package: + yum: name: - - docker - git - jq - gettext @@ -41,22 +67,22 @@ - gcc - make - perl - # - p7zip + #- p7zip - nodejs - state: present + state: latest - name: Install Java17 - package: - name: jdk-17-headless - state: present + yum: + name: java-17-openjdk-headless + state: latest - name: Install pip3 and ansible - package: + yum: name: - python3-pip - ansible-core - glibc-langpack-en - state: present + state: latest - name: Install ansible collections shell: "ansible-galaxy collection install {{ item }}" @@ -110,8 +136,7 @@ mode: "555" - name: Install Python's virtualenv - pip: - name: virtualenv + command: python3 -m venv /home/ec2-user/venv - name: Install Python modules pip: @@ -276,16 +301,6 @@ name: fs.inotify.max_user_watches value: 1048576 - - name: Get disk UUID - command: blkid -s UUID -o value /dev/nvme0n1p1 - register: disk_uuid - - - name: Replace fstab entities with new mount options - lineinfile: - path: /etc/fstab - regexp: "^UUID={{ disk_uuid.stdout }}" - line: "UUID={{ disk_uuid.stdout }} / xfs defaults,noatime,nobarrier,discard 1 1" - - name: Add repo.ci.percona.com to /etc/hosts lineinfile: dest: /etc/hosts diff --git a/build/packer/aws.pkr.hcl b/build/packer/aws.pkr.hcl index d3bd6cd7cf..67e713959c 100644 --- a/build/packer/aws.pkr.hcl +++ b/build/packer/aws.pkr.hcl @@ -39,9 +39,9 @@ source "amazon-ebs" "agent" { iit-billing-tag = "pmm-worker" } launch_block_device_mappings { - device_name = "/dev/xvda" - volume_size = 50 - volume_type = "gp3" + device_name = "/dev/sda1" + volume_size = 50 + volume_type = "gp3" delete_on_termination = true } vpc_filter { @@ -114,7 +114,7 @@ build { user = "ec2-user" ansible_env_vars = ["ANSIBLE_NOCOLOR=True"] extra_arguments = [ - "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-v" + "--ssh-extra-args", "-o HostKeyAlgorithms=+ssh-rsa -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null", "-vvv" ] playbook_file = "./ansible/agent-aws.yml" }