Nomadder 119 (#141)

* nomadder-119 Docu * nomadder-119 Blackbox exporter * nomadder-119 Vector static ip * nomadder-119 BUGFIX: is_master_0 evaluation * nomadder-119 version_nats_server 2.9.16 > 2.9.17 version_nats_prometheus_exporter 0.11.0 > 0.24.0 version_vector_agent 0.29.1 > 0.30.0 nomad_version: 1.5.5 > 1.5.6 version_protoc 22.3 > 23.1 version_proto_buf_cli 1.17.0 > 1.19.0 * nomadder-119 alertmanager integration * nomadder-119 blackbox exporter * nomadder-119 custom envoy proxy * nomadder-119 docu * nomadder-119 vector static ip to nats * nomadder-119 disable host volume for docker driver * nomadder-119 docu certgen * nomadder-119 CI/CD test deployment with jenkins and gitlab * nomadder-119 Test deployment with shared alloc folder * nomadder-119 sysctl set over docker driver * nomadder-119 Delete obsolete script * nomadder-119 Comment * nomadder-119 version_protoc 23.1 > 23.2 * nomadder-119 workaround issue #138 * nomadder-119 refactor systemd nomad restart * nomadder-119 fix docu * nomadder-119 Grafana dashboards and alarms * nomadder-119 Grafana dashboards and alarms --------- Co-authored-by: VURU <[email protected]>
suikast42 · Jun 14, 2023 · a8251bf · a8251bf
1 parent c570f30
commit a8251bf
Show file tree

Hide file tree

Showing 116 changed files with 1,795 additions and 36,158 deletions.
diff --git a/.envrc b/.envrc
@@ -7,3 +7,6 @@ export ANSIBLE_CONFIG="$ENVIRONMENT/../ansible.cfg"
 export ANSIBLE_INVENTORY="$ENVIRONMENT/inventory/hosts.ini"
 export ANSIBLE_DEBUG=False
 export DOCKER_CERT_PATH="$ENVIRONMENT/docker_client"
+
+export PULL_REGISTRY=registry.cloud.private
+export PUSH_REGISTRY=10.21.21.41:5001
diff --git a/ansible/setup/consul/handlers/generate_consul_certs.yml b/ansible/setup/consul/handlers/generate_consul_certs.yml
@@ -63,7 +63,7 @@
         src: ../templates/cert/consul_client.j2
         dest: "{{cfssl_working_dir}}/consul_client.json"
 
-    - name: "Generate consul server certificate(s) in  on master 0"
+    - name: "Generate consul server certificate(s)  on master 0"
       register: cfsslgen
       when: is_master_host
       failed_when: cfsslgen.rc != 0
@@ -77,7 +77,7 @@
       args:
         chdir: "{{cfssl_working_dir}}"
 
-    - name: "Generate consul client certificate(s) in  on master 0"
+    - name: "Generate consul client certificate(s)  on master 0"
       register: cfsslgen
       when: is_worker_host
       failed_when: cfsslgen.rc != 0

diff --git a/ansible/setup/dnsmasq/templates/dnsmasq.j2 b/ansible/setup/dnsmasq/templates/dnsmasq.j2
@@ -1,5 +1,4 @@
-listen-address=127.0.0.1
-listen-address={{ ansible_eth1.ipv4.address }}
+listen-address=127.0.0.1, {{ ansible_eth1.ipv4.address }}
 
 # Enable forward lookup of the 'consul' domain:
 server=/consul/127.0.0.1#8600

diff --git a/ansible/setup/docker/handlers/generate_docker_certs.yml b/ansible/setup/docker/handlers/generate_docker_certs.yml
@@ -47,7 +47,7 @@
       file:
         path: "{{docker_cert_path}}"
         state: directory
-        mode: '0400'
+        mode: '0755'
         group: docker
 
 
@@ -73,15 +73,15 @@
         dest: "{{cfssl_working_dir}}/docker-server.json"
 
     - name: Template docker-client.json
-      when: is_build_host
+#      when: is_build_host
       delegate_to: "{{masters[0]}}"
       template:
         force: yes
         src: ../templates/cert/docker-client.j2
         dest: "{{cfssl_working_dir}}/docker-client.json"
 
 
-    - name: "Generate docker server certificate(s) in  on master 0"
+    - name: "Generate docker server certificate(s)  on master 0"
       register: cfsslgen
       failed_when: cfsslgen.rc != 0
       delegate_to: "{{masters[0]}}"
@@ -94,8 +94,8 @@
       args:
         chdir: "{{cfssl_working_dir}}"
 
-    - name: "Generate docker client certificate(s) in on master 0"
-      when: is_build_host
+    - name: "Generate docker client certificate(s) on master 0"
+#      when: is_build_host
       register: cfsslgen
       failed_when: cfsslgen.rc != 0
       delegate_to: "{{masters[0]}}"
@@ -111,10 +111,12 @@
     - name: "Copy docker server certs"
       include_tasks: cert_install/tasks/copy_from_master_0.yml
       loop:
-        - { src: '{{cfssl_working_dir}}/docker-server.pem', dest: '{{docker_server_cert}}' ,mode: '0400'}
-        - { src: '{{cfssl_working_dir}}/docker-server-key.pem', dest: '{{docker_server_cert_key}}',mode: '0400'}
+        - { src: '{{cfssl_working_dir}}/docker-server.pem', dest: '{{docker_server_cert}}' ,mode: '0644'}
+        - { src: '{{cfssl_working_dir}}/docker-server-key.pem', dest: '{{docker_server_cert_key}}',mode: '0644'}
+        - { src: '{{cfssl_working_dir}}/docker-client.pem', dest: '{{docker_client_cert}}' ,mode: '0644'}
+        - { src: '{{cfssl_working_dir}}/docker-client-key.pem', dest: '{{docker_client_cert_key}}',mode: '0644'}
 
-    - name: "Copy docker client certs"
+    - name: "Copy docker client certs from build host to localhost"
       when: is_build_host
       run_once: true
       include_tasks: cert_install/tasks/copy_from_master_0.yml
@@ -138,4 +140,6 @@
     - cfssl_working_dir: "{{cert_path_master_0}}/{{host_name}}/docker"
     - docker_cert_path: "{{base_cert_dir}}/docker"
     - docker_server_cert: "{{docker_cert_path}}/docker-server.pem"
-    - docker_server_cert_key: "{{docker_cert_path}}/docker-server-key.pem"
+    - docker_server_cert_key: "{{docker_cert_path}}/docker-server-key.pem"
+    - docker_client_cert: "{{docker_cert_path}}/docker-client.pem"
+    - docker_client_cert_key: "{{docker_cert_path}}/docker-client-key.pem"
diff --git a/ansible/setup/docker/tasks/install_docker.yml b/ansible/setup/docker/tasks/install_docker.yml
@@ -114,7 +114,7 @@
   become: true
   when: not dockerInstalled.stat.exists or update_certificates|bool == true
   block:
-    - name: install_cert handler nomad
+    - name: install_cert handler docker
       include_tasks: handlers/generate_docker_certs.yml
 
 

diff --git a/ansible/setup/master_vault/handlers/generate_vault_certs.yml b/ansible/setup/master_vault/handlers/generate_vault_certs.yml
@@ -63,7 +63,7 @@
         dest: "{{cfssl_working_dir}}/vault-server.json"
 
 
-    - name: "Generate vault server certificate(s) in  on master 0"
+    - name: "Generate vault server certificate(s)  on master 0"
       register: cfsslgen
       failed_when: cfsslgen.rc != 0
       delegate_to: "{{masters[0]}}"

diff --git a/ansible/setup/nomad/handlers/generate_nomad_certs.yml b/ansible/setup/nomad/handlers/generate_nomad_certs.yml
@@ -62,12 +62,12 @@
         path: "{{cfssl_working_dir}}"
 
 
-    - name: Template nomad_server.json
+    - name: Template nomad_agent.json
       delegate_to: "{{masters[0]}}"
       template:
         force: yes
-        src: ../templates/cert/nomad_server.j2
-        dest: "{{cfssl_working_dir}}/nomad_server.json"
+        src: ../templates/cert/nomad_agent.j2
+        dest: "{{cfssl_working_dir}}/nomad_agent.json"
 
     - name: Template nomad_cli.json
       delegate_to: "{{masters[0]}}"
@@ -76,31 +76,23 @@
         src: ../templates/cert/nomad_cli.j2
         dest: "{{cfssl_working_dir}}/nomad_cli.json"
 
-    - name: Template nomad_client.json
-      delegate_to: "{{masters[0]}}"
-      template:
-        force: yes
-        src: ../templates/cert/nomad_client.j2
-        dest: "{{cfssl_working_dir}}/nomad_client.json"
-
     - name: Template nomad_ingress.j2
       delegate_to: "{{masters[0]}}"
       template:
         force: yes
         src: ../templates/cert/nomad_ingress.j2
         dest: "{{cfssl_working_dir}}/nomad_ingress.json"
 
-    - name: "Generate nomad server certificate(s) in  on master 0"
+    - name: "Generate nomad server certificate(s)"
       register: cfsslgen
-      when: is_master_host
       failed_when: cfsslgen.rc != 0
       delegate_to: "{{masters[0]}}"
       shell: |
         cfssl gencert \
           -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \
           -ca-key {{cert_path_master_0}}/cluster-ca/cluster-ca-key.pem \
           -config {{cert_path_master_0}}/cfssl/config.json \
-          -profile peer nomad_server.json | cfssljson -bare nomad
+          -profile peer nomad_agent.json | cfssljson -bare nomad
 
         cfssl gencert \
           -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \
@@ -111,21 +103,6 @@
       args:
         chdir: "{{cfssl_working_dir}}"
 
-    - name: "Generate nomad client certificate(s) in  on master 0"
-      register: cfsslgen
-      when: is_worker_host
-      failed_when: cfsslgen.rc != 0
-      delegate_to: "{{masters[0]}}"
-      shell: |
-        cfssl gencert \
-          -ca {{cert_path_master_0}}/cluster-ca/cluster-ca.pem \
-          -ca-key {{cert_path_master_0}}/cluster-ca/cluster-ca-key.pem \
-          -config {{cert_path_master_0}}/cfssl/config.json \
-          -profile peer nomad_client.json | cfssljson -bare nomad
-
-      args:
-        chdir: "{{cfssl_working_dir}}"
-
     - name: "Generate nomad ingress certificate(s) in  on master 0"
       register: cfsslgen
       when: is_worker_host
@@ -155,6 +132,8 @@
       loop:
         - { src: '{{cfssl_working_dir}}/nomad.pem', dest: '{{nomad_cert}}' ,mode: '0644' }
         - { src: '{{cfssl_working_dir}}/nomad-key.pem', dest: '{{nomad_cert_key}}',mode: '0644' }
+        - { src: '{{cfssl_working_dir}}/nomad-cli.pem', dest: '{{nomad_cert_cli}}' ,mode: '0644'}
+        - { src: '{{cfssl_working_dir}}/nomad-cli-key.pem', dest: '{{nomad_cert_cli_key}}',mode: '0644'}
         - { src: '{{cfssl_working_dir}}/nomad_ingress.pem', dest: '{{nomad_cert_ingress}}' ,mode: '0644' }
         - { src: '{{cfssl_working_dir}}/nomad_ingress-key.pem', dest: '{{nomad_cert_ingress_key}}',mode: '0644' }
 
@@ -183,8 +162,9 @@
     - data_center: "nomadder1"
     - nomad_servers:
       - "127.0.0.1"
+      - "{{host_ip}}"
       - "server.global.nomad"
       - "localhost"
-      - "{{nomad_dns}}"
-      - "server.{{data_center}}.nomad"
-      - "server.{{host_name}}.{{data_center}}.nomad"
+      - "{{host_name}}.node.{{data_center}}"
+      - "{{host_name}}.node.{{data_center}}.consul"
+
diff --git a/ansible/setup/nomad/tasks/copy_nomad_config_files.yml b/ansible/setup/nomad/tasks/copy_nomad_config_files.yml
@@ -23,11 +23,3 @@
         dest: "{{nomad_conf_dir}}/client.hcl"
         owner: "{{ nomad_user }}"
       register: cfgClient
-
-
-    - name: "restart nomad service"
-      become: true
-      when: nomad_service.stat.exists and ( (cfgClient is defined and  cfgClient.changed) or (cfgServer is defined and cfgServer.changed))
-      service:
-        name: nomad
-        state: restarted
diff --git a/ansible/setup/nomad/tasks/main.yml b/ansible/setup/nomad/tasks/main.yml
@@ -37,6 +37,52 @@
       - name: Nomad service
         include_tasks: nomad_systemd_install.yml
 
+      - name: Reload systemd
+        become: true
+        ansible.builtin.systemd:
+          daemon_reload: yes
+
+      - name: stop nomad service
+        become: true
+        failed_when: false
+        service:
+          name: nomad
+          state: stopped
+
+      - name: start nomad service
+        become: true
+        service:
+          name: nomad
+          state: started
+          enabled: yes
+        register: nomadservice
+        until: nomadservice.status.ActiveState == "active"
+        retries: 10
+        delay: 5
+
+      - name: Enable Memory Oversubscription
+        become: false
+        when: is_master_host
+        run_once: true
+        shell: |
+          curl  -s  -N \
+          --cacert "${NOMAD_CACERT}" \
+          --cert "${NOMAD_CLIENT_CERT}" \
+          --key "${NOMAD_CLIENT_KEY}" \
+          "${NOMAD_ADDR}/v1/operator/scheduler/configuration" |\
+          jq '.SchedulerConfig | .SchedulerAlgorithm= "spread" |  .MemoryOversubscriptionEnabled=true | .PreemptionConfig.SysBatchSchedulerEnabled=true | .PreemptionConfig.BatchSchedulerEnabled=true  | .PreemptionConfig.ServiceSchedulerEnabled=true'   |\
+          curl \
+          --cacert "${NOMAD_CACERT}" \
+          --cert "${NOMAD_CLIENT_CERT}" \
+          --key "${NOMAD_CLIENT_KEY}" \
+          -X PUT $NOMAD_ADDR/v1/operator/scheduler/configuration -d @-
+        changed_when: false
+        environment:
+          - NOMAD_ADDR: "https://localhost:4646"
+          - NOMAD_CACERT: "{{cluster_intermediate_ca_bundle}}"
+          - NOMAD_CLIENT_CERT: "{{nomad_cert}}"
+          - NOMAD_CLIENT_KEY: "{{nomad_cert_key}}"
+
   - name: nomad update
     when: update_nomad|bool == true
     block:
@@ -49,6 +95,25 @@
       - name: Nomad service
         include_tasks: nomad_systemd_install.yml
 
+      - name: Reload systemd
+        ansible.builtin.systemd:
+          daemon_reload: yes
+
+      - name: stop nomad service
+        failed_when: false
+        service:
+          name: nomad
+          state: stopped
+
+      - name: start nomad service
+        service:
+          name: nomad
+          state: started
+          enabled: yes
+        register: nomadservice
+        until: nomadservice.status.ActiveState == "active"
+        retries: 10
+        delay: 5
   vars:
     consul_cert_path: "{{base_cert_dir}}/consul"
     consul_cert: "{{consul_cert_path}}/consul.pem"
@@ -69,5 +134,4 @@
       - "server.global.nomad"
       - "localhost"
       - "{{nomad_dns}}"
-      - "server.{{data_center}}.nomad"
-      - "server.{{host_name}}.{{data_center}}.nomad"
+      - "{{host_name}}.node.{{data_center}}.consul"
diff --git a/ansible/setup/nomad/tasks/nomad_systemd_install.yml b/ansible/setup/nomad/tasks/nomad_systemd_install.yml
@@ -12,78 +12,39 @@
             dest: /etc/systemd/system/nomad.eligtion.service
           register: serviceconfig_eligtion
 
+        - name: Template nomad_kill_pause_containers.sh.j2
+          template:
+            force: yes
+            src: service/nomad_kill_pause_containers.sh.j2
+            dest: "{{nomad_conf_dir}}/nomad_kill_pause_containers.sh"
+          register: template_kill_pause
+
         - name: Template nomad_node_drain.sh.j2
           template:
             force: yes
             src: service/nomad_node_drain.sh.j2
             dest: "{{nomad_conf_dir}}/nomad_node_drain.sh"
+          register: template_node_drain
 
         - name: "Changing perm of {{nomad_conf_dir}}/nomad_node_drain.sh"
           file: dest="{{nomad_conf_dir}}/nomad_node_drain.sh" mode=+x
 
+        - name: "Changing perm of {{nomad_conf_dir}}/nomad_kill_pause_containers.sh"
+          file: dest="{{nomad_conf_dir}}/nomad_kill_pause_containers.sh" mode=+x
+
         - name: enable nomad eligtion service
           service:
             name: nomad.eligtion
             enabled: yes
 
-        - name: Reload systemd
-          when: serviceconfig_eligtion.changed
-          ansible.builtin.systemd:
-            daemon_reload: yes
-
-
-    - name: Template nomad.service.server
+    - name: Template nomad.service
       template:
         force: yes
         src: service/nomad.service.j2
         dest: /etc/systemd/system/nomad.service
       register: serviceconfig
 
 
-    - name: stop nomad service
-      when: serviceconfig.changed
-      failed_when: false
-      service:
-        name: nomad
-        state: stopped
-
-    - name: Reload systemd
-      when: serviceconfig.changed
-      ansible.builtin.systemd:
-        daemon_reload: yes
-
-    - name: start nomad service
-      service:
-        name: nomad
-        state: started
-        enabled: yes
-      register: nomadservice
-      until: nomadservice.status.ActiveState == "active"
-      retries: 10
-      delay: 5
-
-    - name: Enable Memory Oversubscription
-      become: false
-      when: is_master_host
-      run_once: true
-      shell: |
-        curl  -s  -N \
-        --cacert "${NOMAD_CACERT}" \
-        --cert "${NOMAD_CLIENT_CERT}" \
-        --key "${NOMAD_CLIENT_KEY}" \
-        "${NOMAD_ADDR}/v1/operator/scheduler/configuration" |\
-        jq '.SchedulerConfig | .SchedulerAlgorithm= "spread" |  .MemoryOversubscriptionEnabled=true | .PreemptionConfig.SysBatchSchedulerEnabled=true | .PreemptionConfig.BatchSchedulerEnabled=true  | .PreemptionConfig.ServiceSchedulerEnabled=true'   |\
-        curl \
-        --cacert "${NOMAD_CACERT}" \
-        --cert "${NOMAD_CLIENT_CERT}" \
-        --key "${NOMAD_CLIENT_KEY}" \
-        -X PUT $NOMAD_ADDR/v1/operator/scheduler/configuration -d @-
-      changed_when: false
-      environment:
-        - NOMAD_ADDR: "https://localhost:4646"
-        - NOMAD_CACERT: "{{cluster_intermediate_ca_bundle}}"
-        - NOMAD_CLIENT_CERT: "{{nomad_cert}}"
-        - NOMAD_CLIENT_KEY: "{{nomad_cert_key}}"
 
 #     failed_when: install.rc > 2