DO NOT MERGE: run release tests on tag v0.42.0

.github/workflows/ci.yaml

@@ -30,11 +30,26 @@ on:
         description: "Run release tests in vlab/hlab"
         required: false
         default: false
+      release_test_regex:
+        type: string
+        description: "Run only release tests matched by regex"
+        required: false
+        default: ""
+      release_test_invert_regex:
+        type: boolean
+        description: "Invert regex match"
+        required: false
+        default: false
       debug_enabled:
         type: boolean
         description: "Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)"
         required: false
         default: false
+      pause_on_fail:
+        type: boolean
+        description: "Pause release tests on failure for debugging"
+        required: false
+        default: false
 
 permissions:
   contents: read
@@ -67,6 +82,7 @@ jobs:
         with:
           # keep in sync with hack/tools.just
           version: v1.64.8
+          install-mode: goinstall
 
       - name: Build all
         run: |
@@ -216,7 +232,7 @@ jobs:
     needs:
       - test-build
 
-    name: "${{ matrix.hybrid && 'hlab' || 'vlab' }}-${{ matrix.fabricmode == 'spine-leaf' && 'sl' || 'cc' }}-${{ matrix.mesh && 'mesh-' || '' }}${{ matrix.gateway && 'gw-' || '' }}${{ matrix.includeonie && 'onie-' || '' }}${{ matrix.buildmode }}-${{ matrix.vpcmode }}"
+    name: "${{ matrix.hybrid && 'hlab' || 'vlab' }}-${{ matrix.fabricmode == 'spine-leaf' && 'sl' || 'cc' }}-${{ matrix.mesh && 'mesh-' || '' }}${{ matrix.gateway && 'gw-' || '' }}${{ matrix.includeonie && 'onie-' || '' }}${{ matrix.buildmode }}-${{ matrix.vpcmode }}${{ (inputs.releasetest == true || contains(github.event.pull_request.labels.*.name, 'ci:+release')) && '-rt' || '' }}"
 
     uses: ./.github/workflows/run-vlab.yaml
     with:
@@ -230,7 +246,11 @@ jobs:
       buildmode: ${{ matrix.buildmode }}
       vpcmode: ${{ matrix.vpcmode }}
       releasetest: ${{ inputs.releasetest == true || contains(github.event.pull_request.labels.*.name, 'ci:+release') }}
+      release_test_regex: ${{ inputs.release_test_regex || '' }}
+      release_test_invert_regex: ${{ inputs.release_test_invert_regex == true }}
       hybrid: ${{ matrix.hybrid }}
+      debug: ${{ inputs.debug_enabled || false }}
+      pause_on_fail: ${{ inputs.pause_on_fail || false }}
 
     strategy:
       fail-fast: false
@@ -367,11 +387,12 @@ jobs:
       - name: Prepare debug artifacts
         if: ${{ always() }}
         run: |
-          mkdir -p _debug/0-before
+          mkdir -p _debug/0-before/serial
           cp versions.txt _debug/0-before || true
           cp result/diagram.* _debug/0-before || true
           cp vlab.hhs _debug/0-before || true
           cp -r show-tech-output _debug/0-before || true
+          find ./vlab/vms -type f -name serial.log -exec bash -c 'cp $0 _debug/0-before/serial/$(basename $(dirname $0)).log' {} \; || true
           cp .zot/log _debug/zot.log || true
 
       - name: Build current hhfab
@@ -392,11 +413,12 @@ jobs:
       - name: Prepare debug artifacts
         if: ${{ always() }}
         run: |
-          mkdir -p _debug/1-curr
+          mkdir -p _debug/1-curr/serial
           cp versions.txt _debug/1-curr || true
           cp result/diagram.* _debug/1-curr || true
           cp vlab.hhs _debug/1-curr || true
           cp -r show-tech-output _debug/1-curr || true
+          find ./vlab/vms -type f -name serial.log -exec bash -c 'cp $0 _debug/1-curr/serial/$(basename $(dirname $0)).log' {} \; || true
           cp .zot/log _debug/zot.log || true
 
       - name: hhfab vlab up after upgrade
@@ -413,11 +435,12 @@ jobs:
       - name: Prepare debug artifacts
         if: ${{ always() }}
         run: |
-          mkdir -p _debug/2-after
+          mkdir -p _debug/2-after/serial
           cp versions.txt _debug/2-after || true
           cp result/diagram.* _debug/2-after || true
           cp vlab.hhs _debug/2-after || true
           cp -r show-tech-output _debug/2-after || true
+          find ./vlab/vms -type f -name serial.log -exec bash -c 'cp $0 _debug/2-after/serial/$(basename $(dirname $0)).log' {} \; || true
           cp .zot/log _debug/zot.log || true
           mv _debug fab-${{ github.run_id }}-${{ env.slug }}
 

.github/workflows/run-vlab.yaml

@@ -52,6 +52,16 @@ on:
         type: boolean
         required: false
         default: false
+      release_test_regex:
+        description: "Regex pattern to filter release tests"
+        type: string
+        required: false
+        default: ""
+      release_test_invert_regex:
+        description: "Invert regex selection"
+        type: boolean
+        required: false
+        default: false
       hybrid:
         description: "Enable hybrid mode (use env with physical switches)"
         type: boolean
@@ -62,13 +72,18 @@ on:
         type: boolean
         required: false
         default: false
+      pause_on_fail:
+        description: "Pause release tests on failure for debugging"
+        type: boolean
+        required: false
+        default: false
 
 permissions:
   contents: read
 
 env:
   # global workflow configs
-  slug: "${{ inputs.hybrid && 'hlab' || 'vlab' }}-${{ inputs.fabricmode == 'spine-leaf' && 'sl' || 'cc' }}-${{ inputs.mesh && 'mesh-' || '' }}${{ inputs.gateway && 'gw-' || '' }}${{ inputs.includeonie && 'onie-' || '' }}${{ inputs.buildmode }}-${{ inputs.vpcmode }}"
+  slug: "${{ inputs.hybrid && 'hlab' || 'vlab' }}-${{ inputs.fabricmode == 'spine-leaf' && 'sl' || 'cc' }}-${{ inputs.mesh && 'mesh-' || '' }}${{ inputs.gateway && 'gw-' || '' }}${{ inputs.includeonie && 'onie-' || '' }}${{ inputs.buildmode }}-${{ inputs.vpcmode }}${{ inputs.releasetest && '-rt' || '' }}"
   # env vars to configure hhfab
   HHFAB_REG_REPO: 127.0.0.1:30000
   HHFAB_VLAB_COLLECT: true
@@ -189,6 +204,9 @@ jobs:
             ${{ !inputs.gateway && '--ready=setup-peerings' || '' }} \
             --ready=test-connectivity \
             ${{ inputs.releasetest && '--ready=release-test' || '' }} \
+            ${{ inputs.release_test_regex && format('--release-test-regex="{0}"', inputs.release_test_regex) || '' }} \
+            ${{ inputs.release_test_invert_regex && '--release-test-invert-regex' || '' }} \
+            ${{ inputs.pause_on_fail && '--pause-on-fail' || '' }} \
             --ready=exit
 
       - name: Upload Release Test Results
@@ -201,12 +219,13 @@ jobs:
       - name: Prepare debug artifacts
         if: ${{ always() }}
         run: |
-          mkdir _debug
+          mkdir -p _debug/serial
           cp versions.txt _debug/ || true
           cp result/diagram.* _debug/ || true
           cp vlab.hhs _debug/ || true
           cp -r show-tech-output _debug/ || true
           cp .zot/log _debug/zot.log || true
+          find ./vlab/vms -type f -name serial.log -exec bash -c 'cp $0 _debug/serial/$(basename $(dirname $0)).log' {} \; || true
           mv _debug fab-${{ github.run_id }}-${{ env.slug }}
 
       - name: Upload debug artifacts

cmd/hhfab/main.go

@@ -765,6 +765,11 @@ func Run(ctx context.Context) error {
 								Usage: "exit on first error",
 								Value: true,
 							},
+							&cli.BoolFlag{
+								Name:    FlagPauseOnFail,
+								Aliases: []string{"p"},
+								Usage:   "pause testing on each scenario failure (for troubleshooting)",
+							},
 							&cli.StringSliceFlag{
 								Name:    FlagNameReady,
 								Aliases: []string{"r"},
@@ -780,6 +785,16 @@ func Run(ctx context.Context) error {
 								Name:  FlagNameVPCMode,
 								Usage: "VPC mode to be used for on-ready commands: empty is default (l2vni), l3vni, etc.",
 							},
+							&cli.StringSliceFlag{
+								Name:    "release-test-regex",
+								Aliases: []string{"rt-regex"},
+								Usage:   "regex pattern to filter release tests (used when --ready=release-test)",
+							},
+							&cli.BoolFlag{
+								Name:    "release-test-invert-regex",
+								Aliases: []string{"rt-invert"},
+								Usage:   "invert regex selection for release tests (used when --ready=release-test)",
+							},
 						}),
 						Before: before(false),
 						Action: func(c *cli.Context) error {
@@ -798,6 +813,9 @@ func Run(ctx context.Context) error {
 									OnReady:            c.StringSlice(FlagNameReady),
 									CollectShowTech:    c.Bool(FlagNameCollectShowTech),
 									VPCMode:            vpcapi.VPCMode(handleL2VNI(c.String(FlagNameVPCMode))),
+									PauseOnFail:        c.Bool(FlagPauseOnFail),
+									ReleaseTestRegexes: c.StringSlice("release-test-regex"),
+									ReleaseTestInvert:  c.Bool("release-test-invert-regex"),
 								},
 							}); err != nil {
 								return fmt.Errorf("running VLAB: %w", err)

go.mod

@@ -42,7 +42,7 @@ require (
 	github.com/urfave/cli/v2 v2.27.7
 	github.com/vbauerster/mpb/v8 v8.10.2
 	github.com/vishvananda/netlink v1.3.1
-	go.githedgehog.com/fabric v0.88.2
+	go.githedgehog.com/fabric v0.88.1
 	go.githedgehog.com/gateway v0.17.0
 	golang.org/x/crypto v0.41.0
 	golang.org/x/mod v0.27.0

go.sum

@@ -909,8 +909,8 @@ github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM=
 github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
 go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
 go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
-go.githedgehog.com/fabric v0.88.2 h1:rs51cW27ilmdYjwkZ2/6baBBZtw+7aRkFg53/SXPxB0=
-go.githedgehog.com/fabric v0.88.2/go.mod h1:oSHoCN+J8xiJMY3aCUmv3Sbtgw9/qOlcAVhPmliZX8Y=
+go.githedgehog.com/fabric v0.88.1 h1:yxS8mq5aPe+R5SPH8LNJpU6P5LNwTNES80dhORI8BZo=
+go.githedgehog.com/fabric v0.88.1/go.mod h1:oSHoCN+J8xiJMY3aCUmv3Sbtgw9/qOlcAVhPmliZX8Y=
 go.githedgehog.com/fabric-bcm-ygot v0.5.2-4.5.0 h1:BbK131yz1T6ldx7xsREa/A2v5bJdLcs1gVlxDujdEKg=
 go.githedgehog.com/fabric-bcm-ygot v0.5.2-4.5.0/go.mod h1:XWCdFbSbGP6nlg90c/922VblpHOTw07TCwx1yUHPkzc=
 go.githedgehog.com/gateway v0.17.0 h1:revvLmkdtDwro1CDBTVUQPJtusCb5H6+RbpmXqeeXsM=

pkg/fab/README.md

@@ -5,7 +5,7 @@
 Stable version is listed on the [Flatcar releases page](https://www.flatcar.org/releases).
 
 ```bash
-export FLATCAR_VERSION="v4230.2.1"
+export FLATCAR_VERSION="v4230.2.0"
 export FLATCAR_VERSION_UPSTREAM="${FLATCAR_VERSION:1}"
 
 wget "https://stable.release.flatcar-linux.net/amd64-usr/${FLATCAR_VERSION_UPSTREAM}/flatcar_production_qemu_image.img"

pkg/fab/versions.go

@@ -15,7 +15,7 @@ import (
 
 var (
 	FabricatorVersion = meta.Version(version.Version)
-	FabricVersion     = meta.Version("v0.88.2")
+	FabricVersion     = meta.Version("v0.88.1")
 	GatewayVersion    = meta.Version("v0.17.0")
 	DataplaneVersion  = meta.Version("main.x86_64-unknown-linux-gnu.debug.f27f76cd91213cf4dc85d0dab95e7c70ede30efc")
 	FRRVersion        = meta.Version("0ba323e489ea2baf3f85fc42ff23aff674a25690.debug")
@@ -46,8 +46,8 @@ var Versions = fabapi.Versions{
 		Ctl:            FabricatorVersion,
 		NodeConfig:     FabricatorVersion,
 		Pause:          "3.6", // wait image from k3s // TODO embed wait into node-config image?
-		ControlUSBRoot: "v4230.2.1-hh3",
-		Flatcar:        "v4230.2.1",
+		ControlUSBRoot: "v4152.2.3-hh1",
+		Flatcar:        "v4152.2.3",
 	},
 	Fabric: fabapi.FabricVersions{
 		API:        FabricVersion,
@@ -91,7 +91,7 @@ var Versions = fabapi.Versions{
 	},
 	VLAB: fabapi.VLABVersions{
 		ONIE:    "v0.2.0",
-		Flatcar: "v4230.2.1",
+		Flatcar: "v4152.2.3",
 	},
 }
 

pkg/hhfab/release.go

@@ -24,6 +24,7 @@ import (
 	"go.githedgehog.com/fabric/pkg/hhfctl"
 	"go.githedgehog.com/fabric/pkg/util/pointer"
 	"golang.org/x/sync/errgroup"
+	"golang.org/x/term"
 	kmetav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kclient "sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -2378,12 +2379,24 @@ func printSuiteResults(ts *JUnitTestSuite) {
 }
 
 func pauseOnFail() {
-	// pause until the user presses enter
-	slog.Warn("Test failed, pausing execution. Note that reverts might still need to apply, so if you intend to continue, please make sure to leave the environment in the same state as you found it")
-	slog.Info("Press enter to continue...")
-	var input string
-	_, _ = fmt.Scanln(&input)
-	slog.Info("Continuing...")
+	if !term.IsTerminal(int(os.Stdin.Fd())) {
+		// CI environment - pause for a long time to allow debugging
+		pauseDuration := 60 * time.Minute
+		slog.Warn("Running in non-interactive environment (CI)")
+		slog.Info("Pausing for debugging", "duration", pauseDuration)
+		slog.Info("You can connect to debug the VLAB state during this pause")
+		slog.Info("The test will automatically continue after the pause duration")
+
+		time.Sleep(pauseDuration)
+		slog.Info("Pause duration expired, continuing...")
+	} else {
+		// pause until the user presses enter
+		slog.Warn("Test failed, pausing execution. Note that reverts might still need to apply, so if you intend to continue, please make sure to leave the environment in the same state as you found it")
+		slog.Info("Press enter to continue...")
+		var input string
+		_, _ = fmt.Scanln(&input)
+		slog.Info("Continuing...")
+	}
 }
 
 func doRunTests(ctx context.Context, testCtx *VPCPeeringTestCtx, ts *JUnitTestSuite) (*JUnitTestSuite, error) {

pkg/hhfab/vlabrunner.go

@@ -98,6 +98,9 @@ type VLABRunOpts struct {
 	OnReady            []string
 	CollectShowTech    bool
 	VPCMode            vpcapi.VPCMode
+	PauseOnFail        bool
+	ReleaseTestRegexes []string
+	ReleaseTestInvert  bool
 }
 
 type OnReady string
@@ -584,6 +587,10 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("reinstalling switches: %w", err)
 					}
 				case OnReadySetupVPCs:
@@ -603,6 +610,10 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("setting up VPCs: %w", err)
 					}
 				case OnReadySetupPeerings:
@@ -623,6 +634,10 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("setting up peerings: %w", err)
 					}
 				case OnReadyTestConnectivity:
@@ -637,6 +652,10 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("testing connectivity: %w", err)
 					}
 				case OnReadyExit:
@@ -655,6 +674,10 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("waiting: %w", err)
 					}
 				case OnReadyInspect:
@@ -667,13 +690,20 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 
 						c.CollectVLABDebug(ctx, vlab, opts)
 
+						if opts.PauseOnFail {
+							pauseOnFail()
+						}
+
 						return fmt.Errorf("inspecting: %w", err)
 					}
 				case OnReadyReleaseTest:
 					if err := c.ReleaseTest(ctx, ReleaseTestOpts{
 						ResultsFile: "release-test.xml",
 						HashPolicy:  HashPolicyL2And3,
 						VPCMode:     opts.VPCMode,
+						PauseOnFail: opts.PauseOnFail,
+						Regexes:     opts.ReleaseTestRegexes,
+						InvertRegex: opts.ReleaseTestInvert,
 					}); err != nil {
 						slog.Warn("Failed to run release test", "err", err)