nomadder-119 workaround issue #138

ansible/setup/nomad/tasks/nomad_systemd_install.yml

@@ -21,6 +21,16 @@
         - name: "Changing perm of {{nomad_conf_dir}}/nomad_node_drain.sh"
           file: dest="{{nomad_conf_dir}}/nomad_node_drain.sh" mode=+x
 
+        - name: Template nomad_kill_pause_containers.sh.j2
+          template:
+            force: yes
+            src: service/nomad_kill_pause_containers.sh.j2
+            dest: "{{nomad_conf_dir}}/nomad_kill_pause_containers.sh"
+
+        - name: "Changing perm of {{nomad_conf_dir}}/nomad_kill_pause_containers.sh"
+          file: dest="{{nomad_conf_dir}}/nomad_kill_pause_containers.sh" mode=+x
+
+
         - name: enable nomad eligtion service
           service:
             name: nomad.eligtion

ansible/setup/nomad/templates/conf/nomad_client.j2

@@ -30,11 +30,11 @@ client {
   }
   # Either leave_on_interrupt or leave_on_terminate must be set
   # for this to take effect.
-#  drain_on_shutdown {
-#    deadline           = "2m"
-#    force              = false
-#    ignore_system_jobs = false
-#  }
+  drain_on_shutdown {
+    deadline           = "1h"
+    force              = false
+    ignore_system_jobs = false
+  }
   host_volume "ca_cert" {
     path      = "/usr/local/share/ca-certificates/cloudlocal"
     read_only = true

ansible/setup/nomad/templates/service/nomad.service.j2

@@ -36,12 +36,17 @@ Group=root
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nomad agent -config {{nomad_conf_dir}}
 {% if is_worker_host  %}
+# See issue https://github.com/hashicorp/nomad/issues/17299
+# See issue https://github.com/suikast42/nomadder/issues/138
+ExecStartPre={{nomad_conf_dir}}/nomad_kill_pause_containers.sh
 # nomad client have a active setting drain_on_shutdown
 # this drains the node and mark it as ineligible.
 # Make the node eligible again
-#ExecStartPost=/bin/bash -c '(while ! nomad  node eligibility  -enable -self -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad-cli.pem -client-key=/etc/opt/certs/nomad/nomad-cli-key.pem  >/dev/null; do echo "Waiting for port nomad agent runs..."; sleep 1; done); sleep 1'
 ExecStartPost=systemctl restart nomad.eligtion.service
-ExecStop={{nomad_conf_dir}}/nomad_node_drain.sh
+#ExecStartPost=/bin/bash -c '(while ! nomad  node eligibility  -enable -self -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad-cli.pem -client-key=/etc/opt/certs/nomad/nomad-cli-key.pem  >/dev/null; do echo "Waiting for port nomad agent runs..."; sleep 1; done); sleep 1'
+# Use node drain over client config drain_on_shutdown
+# Enable this section if you disable the option drain_on_shutdown
+#ExecStop={{nomad_conf_dir}}/nomad_node_drain.sh
 {% endif %}
 
 KillMode=process
@@ -64,7 +69,7 @@ RestartSec=2
 StartLimitIntervalSec = 10s
 
 # drain_on_shutdown +  30s
-TimeoutStopSec=2m30s
+TimeoutStopSec=1h
 ## StartLimitInterval is used for systemd versions < 230
 # StartLimitInterval = 10s
 

ansible/setup/nomad/templates/service/nomad_kill_pause_containers.sh.j2

@@ -0,0 +1,5 @@
+#!/bin/bash
+CONTAINER_IDS=$(docker ps | grep "amd64")
+if [ -n "$CONTAINER_IDS" ]; then
+    docker kill $(docker ps | grep "amd64" | awk '{ print $1 }')
+fi

ansible/setup/nomad/templates/service/nomad_node_drain.sh.j2

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-if [ ! -f "/home/{{ansible_user}}/notdrain" ] ; then nomad node drain -enable -self -deadline "2m"  -m "Node shutdown" -yes -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad.pem -client-key=/etc/opt/certs/nomad/nomad-key.pem;fi
+if [ ! -f "/home/{{ansible_user}}/notdrain" ] ; then nomad node drain -enable -self -deadline "1h"  -m "Node shutdown" -yes -address=https://localhost:4646 -ca-cert=/usr/local/share/ca-certificates/cloudlocal/cluster-ca-bundle.pem -client-cert=/etc/opt/certs/nomad/nomad.pem -client-key=/etc/opt/certs/nomad/nomad-key.pem;fi

docs/902_cheet_sheet_docker.adoc

@@ -9,4 +9,10 @@ a|[source,shell]
 docker kill $( docker ps \| grep "amd64:3.2"  \|  awk '{ print $1 }')
 ----
 
+|List containers
+a|[source,shell]
+----
+ docker ps | grep "amd64:3.2"  |  awk '{print $4,$5,$1,$10 }'
+----
+
 |===