cluster: prompt for instance restart

Author: dveeden

components/cluster/command/upgrade.go

@@ -14,6 +14,8 @@
 package command
 
 import (
+	"time"
+
 	"github.com/pingcap/tiup/pkg/cluster/spec"
 	"github.com/pingcap/tiup/pkg/utils"
 	"github.com/spf13/cobra"
@@ -23,6 +25,7 @@ func newUpgradeCmd() *cobra.Command {
 	offlineMode := false
 	ignoreVersionCheck := false
 	var tidbVer, tikvVer, pdVer, tsoVer, schedulingVer, tiflashVer, kvcdcVer, dashboardVer, cdcVer, alertmanagerVer, nodeExporterVer, blackboxExporterVer, tiproxyVer string
+	var restartTimeout time.Duration
 
 	cmd := &cobra.Command{
 		Use:   "upgrade <cluster-name> <version>",
@@ -57,7 +60,7 @@ func newUpgradeCmd() *cobra.Command {
 				spec.ComponentNodeExporter:     nodeExporterVer,
 			}
 
-			return cm.Upgrade(clusterName, version, componentVersions, gOpt, skipConfirm, offlineMode, ignoreVersionCheck)
+			return cm.Upgrade(clusterName, version, componentVersions, gOpt, skipConfirm, offlineMode, ignoreVersionCheck, restartTimeout)
 		},
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			switch len(args) {
@@ -89,5 +92,6 @@ func newUpgradeCmd() *cobra.Command {
 	cmd.Flags().StringVar(&nodeExporterVer, "node-exporter-version", "", "Fix the version of node-exporter and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&blackboxExporterVer, "blackbox-exporter-version", "", "Fix the version of blackbox-exporter and no longer follows the cluster version.")
 	cmd.Flags().StringVar(&tiproxyVer, "tiproxy-version", "", "Fix the version of tiproxy and no longer follows the cluster version.")
+	cmd.Flags().DurationVar(&restartTimeout, "restart-timeout", time.Second*0, "Timeout for after upgrade prompt")
 	return cmd
 }

components/dm/command/upgrade.go

@@ -14,6 +14,8 @@
 package command
 
 import (
+	"time"
+
 	"github.com/spf13/cobra"
 )
 
@@ -28,7 +30,7 @@ func newUpgradeCmd() *cobra.Command {
 				return cmd.Help()
 			}
 
-			return cm.Upgrade(args[0], args[1], nil, gOpt, skipConfirm, offlineMode, ignoreVersionCheck)
+			return cm.Upgrade(args[0], args[1], nil, gOpt, skipConfirm, offlineMode, ignoreVersionCheck, time.Second*0)
 		},
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			switch len(args) {

pkg/cluster/manager/patch.go

@@ -105,7 +105,7 @@ func (m *Manager) Patch(name string, packagePath string, opt operator.Options, o
 				return nil
 			}
 			// TBD: should patch be treated as an upgrade?
-			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, base.Version)
+			return operator.Upgrade(ctx, topo, opt, tlsCfg, base.Version, base.Version, nil)
 		}).
 		Build()
 

pkg/cluster/manager/reload.go

@@ -124,7 +124,7 @@ func (m *Manager) Reload(name string, gOpt operator.Options, skipRestart, skipCo
 			return err
 		}
 		b.Func("Upgrade Cluster", func(ctx context.Context) error {
-			return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version, base.Version)
+			return operator.Upgrade(ctx, topo, gOpt, tlsCfg, base.Version, base.Version, nil)
 		})
 	}
 

pkg/cluster/manager/upgrade.go

@@ -39,7 +39,7 @@ import (
 )
 
 // Upgrade the cluster.
-func (m *Manager) Upgrade(name string, clusterVersion string, componentVersions map[string]string, opt operator.Options, skipConfirm, offline, ignoreVersionCheck bool) error {
+func (m *Manager) Upgrade(name string, clusterVersion string, componentVersions map[string]string, opt operator.Options, skipConfirm, offline, ignoreVersionCheck bool, restartTimeout time.Duration) error {
 	if !skipConfirm && strings.ToLower(opt.DisplayMode) != "json" {
 		for _, v := range componentVersions {
 			if v != "" {
@@ -121,11 +121,12 @@ func (m *Manager) Upgrade(name string, clusterVersion string, componentVersions
 	}
 
 	m.logger.Warnf(`%s
-This operation will upgrade %s %s cluster %s to %s:%s`,
+This operation will upgrade %s %s cluster %s (with a concurrency of %d) to %s:%s`,
 		color.YellowString("Before the upgrade, it is recommended to read the upgrade guide at https://docs.pingcap.com/tidb/stable/upgrade-tidb-using-tiup and finish the preparation steps."),
 		m.sysName,
 		color.HiYellowString(base.Version),
 		color.HiYellowString(name),
+		opt.Concurrency,
 		color.HiYellowString(clusterVersion),
 		compVersionMsg)
 	if !skipConfirm {
@@ -317,7 +318,23 @@ This operation will upgrade %s %s cluster %s to %s:%s`,
 			}
 			nopt := opt
 			nopt.Roles = restartComponents
-			return operator.Upgrade(ctx, topo, nopt, tlsCfg, base.Version, clusterVersion)
+			waitFunc := func() {}
+			if restartTimeout.Nanoseconds() > 0 {
+				waitFunc = func() {
+					// A tui.PromptWithTimeout(str, time.Duration) would have been nice
+					ch := make(chan any, 1)
+					go func() {
+						tui.Prompt(fmt.Sprintf("\nPress any key to continue (timeout %s)", restartTimeout))
+						ch <- nil
+					}()
+					select {
+					case <-time.After(restartTimeout):
+						fmt.Printf("\nTimeout, continueing\n")
+					case <-ch:
+					}
+				}
+			}
+			return operator.Upgrade(ctx, topo, nopt, tlsCfg, base.Version, clusterVersion, waitFunc)
 		}).
 		Build()
 

pkg/cluster/operation/upgrade.go

@@ -39,6 +39,8 @@ var (
 	increaseLimitPoint = checkpoint.Register()
 )
 
+type UpgradeWaitFunc func()
+
 // Upgrade the cluster. (actually, it's rolling restart)
 func Upgrade(
 	ctx context.Context,
@@ -47,6 +49,7 @@ func Upgrade(
 	tlsCfg *tls.Config,
 	currentVersion string,
 	targetVersion string,
+	waitFunc UpgradeWaitFunc,
 ) error {
 	roleFilter := set.NewStringSet(options.Roles...)
 	nodeFilter := set.NewStringSet(options.Nodes...)
@@ -189,6 +192,8 @@ func Upgrade(
 			if err := upgradeInstance(ctx, topo, instance, options, tlsCfg); err != nil {
 				return err
 			}
+
+			waitFunc()
 		}
 
 		// process deferred instances
@@ -288,6 +293,8 @@ func upgradeInstance(
 		return err
 	}
 
+
+
 	return nil
 }