automatic life-cycle logic added

CHANGES.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## 5.2.11
+
+* Allow setting `docker_username` to execute the worker node removal scripts on destory of worker nodes.
+
 ## 5.2.10
 
 * `vm.overcommit_memory=1` is to prevent [background saving issues with Redis](https://redis.io/topics/faq#background-saving-fails-with-a-fork-error-under-linux-even-if-i-have-a-lot-of-free-ram)

README.md

@@ -101,7 +101,14 @@ Upgrading a 3 manager swarm needs to be done one at a time to prevent raft conse
 
 ### Upgrading the worker nodes
 
-A future relase of this would utilize auto-scaling for now this needs to be done manually
+A future release of this would utilize auto-scaling for now this needs to be done manually
+
+If `docker_username` is set:
+
+1. Destroy the workers removed from the command line `terraform destroy -target module.docker-swarm.aws_instance.workers[#]`.  `destroy` is used instead of `taint` as [`taint` won't execute the destroy provisioner](https://www.terraform.io/language/resources/provisioners/syntax#destroy-time-provisioners).
+2. Rebuild the workers from the command line `terraform apply`
+
+If `docker_username` is not set:
 
 1. Drain and remove the worker node(s) from the swarm using `ssh <username>@<manager0> sudo /root/bin/rm-workers.sh <nodename[s]>`
 2. Taint the workers that are removed from the command line `terraform taint module.docker-swarm.aws_instance.workers[#]`

variables.tf

@@ -182,3 +182,8 @@ variable "ssh_users" {
   description = "A list of IAM users that will have SSH access when using `iam` for `ssh_authorization_method`"
   default     = []
 }
+
+variable "docker_username" {
+  description = "User on manager0 that can execute the life-cycle operations such as removing a worker node.  The SSH agent on the local device must be used."
+  default     = ""
+}

workers.tf

@@ -94,6 +94,21 @@ resource "aws_instance" "workers" {
   credit_specification {
     cpu_credits = "standard"
   }
+
+  provisioner "remote-exec" {
+    when = destroy
+    inline = [ 
+      "docker node update --availability drain ${self.private_ip}",
+      "sleep 10",
+      "docker node rm --force ${self.private_ip}"
+    ]
+    on_failure = continue
+    connection {
+      type = "ssh"
+      user = var.docker_username
+      host = aws_instance.managers[0].public_ip
+    }
+  }
 }
 
 resource "aws_cloudwatch_metric_alarm" "low-cpu-credit-workers" {