diff --git a/CHANGES.md b/CHANGES.md index a9418c6..9a4f655 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,9 @@ # Change Log +## 5.2.11 + +* Allow setting `docker_username` to execute the worker node removal scripts on destory of worker nodes. + ## 5.2.10 * `vm.overcommit_memory=1` is to prevent [background saving issues with Redis](https://redis.io/topics/faq#background-saving-fails-with-a-fork-error-under-linux-even-if-i-have-a-lot-of-free-ram) diff --git a/README.md b/README.md index 879b8cc..be14bbf 100644 --- a/README.md +++ b/README.md @@ -101,7 +101,14 @@ Upgrading a 3 manager swarm needs to be done one at a time to prevent raft conse ### Upgrading the worker nodes -A future relase of this would utilize auto-scaling for now this needs to be done manually +A future release of this would utilize auto-scaling for now this needs to be done manually + +If `docker_username` is set: + +1. Destroy the workers removed from the command line `terraform destroy -target module.docker-swarm.aws_instance.workers[#]`. `destroy` is used instead of `taint` as [`taint` won't execute the destroy provisioner](https://www.terraform.io/language/resources/provisioners/syntax#destroy-time-provisioners). +2. Rebuild the workers from the command line `terraform apply` + +If `docker_username` is not set: 1. Drain and remove the worker node(s) from the swarm using `ssh @ sudo /root/bin/rm-workers.sh ` 2. Taint the workers that are removed from the command line `terraform taint module.docker-swarm.aws_instance.workers[#]` diff --git a/variables.tf b/variables.tf index a3ef2ff..c3f5f99 100644 --- a/variables.tf +++ b/variables.tf @@ -182,3 +182,8 @@ variable "ssh_users" { description = "A list of IAM users that will have SSH access when using `iam` for `ssh_authorization_method`" default = [] } + +variable "docker_username" { + description = "User on manager0 that can execute the life-cycle operations such as removing a worker node. The SSH agent on the local device must be used." + default = "" +} diff --git a/workers.tf b/workers.tf index ecdca27..b98b51d 100644 --- a/workers.tf +++ b/workers.tf @@ -94,6 +94,21 @@ resource "aws_instance" "workers" { credit_specification { cpu_credits = "standard" } + + provisioner "remote-exec" { + when = destroy + inline = [ + "docker node update --availability drain ${self.private_ip}", + "sleep 10", + "docker node rm --force ${self.private_ip}" + ] + on_failure = continue + connection { + type = "ssh" + user = var.docker_username + host = aws_instance.managers[0].public_ip + } + } } resource "aws_cloudwatch_metric_alarm" "low-cpu-credit-workers" {