@@ -260,7 +260,15 @@ func (a *AutoSpotting) processEventInstance(eventType string, region string, ins
260260 spotTermination := newSpotTermination (region )
261261
262262 if spotTermination .IsInAutoSpottingASG (instanceID , a .config .TagFilteringMode , a .config .FilterByTags ) {
263- err := spotTermination .executeAction (instanceID , a .config .TerminationNotificationAction , eventType )
263+ newInstance , err := a .replaceTerminatingSpotInstance (* instanceID , region )
264+ if err == nil {
265+ log .Printf ("Launched replacement instance for instance %s: %s\n " , * instanceID , * newInstance )
266+ return nil
267+ }
268+
269+ log .Printf ("Error launching replacement instance for instance %s: %s, continued to handle its Spot termination\n " , * instanceID , err .Error ())
270+
271+ err = spotTermination .executeAction (instanceID , a .config .TerminationNotificationAction , eventType )
264272 if err != nil {
265273 log .Printf ("Error executing spot termination/rebalance action: %s\n " , err .Error ())
266274 return err
@@ -438,10 +446,13 @@ func (a *AutoSpotting) handleNewInstanceLaunch(regionName string, instanceID str
438446 }
439447
440448 // Try OnDemand
441- if err := a .handleNewOnDemandInstanceLaunch (r , i ); err != nil {
449+ if err := a .handleNewOnDemandInstanceLaunch (r , i ); ! i .isSpot () && err != nil {
450+ log .Printf ("%s Instance %s couldn't be handled as on-demand instance" , i .region .name , * i .InstanceId )
442451 return err
443452 }
444453
454+ log .Printf ("%s Instance %s couldn't be handled as on-demand instance" , i .region .name , * i .InstanceId )
455+
445456 // Try Spot
446457 // in case we're not triggered by SQS event we do nothing, onDemand event already manage launched spot instance
447458 if len (a .config .sqsReceiptHandle ) > 0 {
@@ -511,9 +522,9 @@ func (a *AutoSpotting) handleNewOnDemandInstanceLaunch(r *region, i *instance) e
511522 }
512523
513524 } else {
514- log .Printf ("%s skipping instance %s: either doesn't belong to an " +
525+ log .Printf ("%s skipping %s instance %s: either doesn't belong to an " +
515526 "enabled ASG or should not be replaced with spot, " ,
516- i .region .name , * i .InstanceId )
527+ i .region .name , * i .InstanceLifecycle , * i . InstanceId )
517528 debug .Printf ("%#v" , i )
518529 }
519530 return nil
@@ -551,3 +562,77 @@ func (a *AutoSpotting) handleNewSpotInstanceLaunch(r *region, i *instance) error
551562 }
552563 return nil
553564}
565+
566+ func (a * AutoSpotting ) replaceTerminatingSpotInstance (instanceID , regionName string ) (* string , error ) {
567+ r := & region {name : regionName , conf : a .config , services : connections {}}
568+
569+ if ! r .enabled () {
570+ return nil , fmt .Errorf ("region %s is not enabled" , regionName )
571+ }
572+
573+ r .services .connect (regionName , a .config .MainRegion )
574+ r .setupAsgFilters ()
575+ r .scanForEnabledAutoScalingGroups ()
576+
577+ log .Println ("Scanning full instance information in" , r .name )
578+ r .determineInstanceTypeInformation (r .conf )
579+
580+ if err := r .scanInstance (aws .String (instanceID )); err != nil {
581+ log .Printf ("%s Couldn't scan instance %s: %s" , regionName ,
582+ instanceID , err .Error ())
583+ return nil , err
584+ }
585+
586+ i := r .instances .get (instanceID )
587+ if i == nil {
588+ log .Printf ("%s Instance %s is missing, skipping..." ,
589+ regionName , instanceID )
590+ return nil , errors .New ("instance missing" )
591+ }
592+ log .Printf ("%s Found instance %s in state %s" ,
593+ i .region .name , * i .InstanceId , * i .State .Name )
594+
595+ if * i .State .Name != "running" {
596+ log .Printf ("%s Instance %s is not in the running state" ,
597+ i .region .name , * i .InstanceId )
598+ return nil , errors .New ("instance not in running state" )
599+ }
600+
601+ asgName := i .getReplacementTargetASGName ()
602+
603+ if asgName == nil {
604+ log .Printf ("Missing the ASG name tag\n " )
605+ return nil , errors .New ("missing ASG name tag" )
606+ }
607+
608+ i .asg = i .region .findEnabledASGByName (* asgName )
609+ i .asg .scanInstances ()
610+ i .asg .loadDefaultConfig ()
611+ i .asg .loadConfigFromTags ()
612+ i .asg .loadLaunchConfiguration ()
613+ i .asg .loadLaunchTemplate ()
614+
615+ newInstanceID , err := i .launchSpotReplacement ()
616+ if err != nil {
617+ fmt .Printf ("Spot Instance launch failed while replacing %s, error: %s, falling back to on-demand\n " , * i .InstanceId , err .Error ())
618+
619+ newInstanceID , err = i .launchReplacement ("on-demand" )
620+ if err != nil {
621+ fmt .Printf ("Instance launch failed while replacing %s, error: %s\n " , * i .InstanceId , err .Error ())
622+ return nil , err
623+ }
624+ }
625+
626+ i .region .scanInstances ()
627+ newInstance := i .region .instances .get (* newInstanceID )
628+
629+ newInstance .swapWithGroupMember (i .asg )
630+
631+ if err = i .asg .waitForInstanceStatus (newInstanceID , "InService" , 5 ); err != nil {
632+ log .Printf ("Instance %s is still not InService, trying to terminate it." ,
633+ * newInstanceID )
634+ newInstance .terminate ()
635+ }
636+
637+ return newInstanceID , nil
638+ }
0 commit comments