Add rollback

.github/workflows/docker-image.yml

@@ -1130,6 +1130,106 @@ jobs:
       - name: Check the capabilities of cap1
         run: if [[ $(sudo podman container inspect cap1 --format {{.EffectiveCaps}} | grep NET_ADMIN | wc -l) = "1" ]] ; then echo "Container successfully launched"; else exit 1; fi
 
+  rollback-validate:
+    runs-on: ubuntu-latest
+    if: >
+      (github.event_name == 'push' || github.event_name == 'schedule') &&
+      (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v'))
+    needs: [ build , pull-and-archive ]
+    steps:
+      - name: pull in podman
+        uses: actions/download-artifact@v1
+        with:
+          name: podman-bins
+          path: bin
+
+      - name: replace
+        run: |
+          chmod +x bin/podman
+          sudo mv bin/podman /usr/bin/podman
+
+      - name: Enable the podman socket
+        run: sudo systemctl enable --now podman.socket
+
+      - uses: actions/checkout@v3
+        with:
+          path: main
+
+      - name: checkout with token
+        uses: actions/checkout@v3
+        with:
+          path: ci-rollback
+          ref: ci-rollback
+
+      - name: Enable the podman socket
+        run: sudo systemctl enable --now podman.socket
+
+      - name: reset and commit working
+        run: |
+          cd ci-rollback
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          rm -rf ./examples/ci-rollback || true
+          mkdir ./examples/ci-rollback
+          cp ./examples/rollback/working.yaml ./examples/ci-rollback/working.yaml
+          git add ./examples
+          git commit -m "add working" -a || true
+          git push -f || true
+
+      - name: pull artifact
+        uses: actions/download-artifact@v1
+        with:
+          name: fetchit-image
+          path: /tmp
+
+      - name: Load the image
+        run: sudo podman load -i /tmp/fetchit.tar
+
+      - name: pull artifact
+        uses: actions/download-artifact@v1
+        with:
+          name: colors
+          path: /tmp
+
+      - name: Load the image
+        run: sudo podman load -i /tmp/colors.tar
+
+      - name: tag the image
+        run: sudo podman tag quay.io/fetchit/fetchit-amd:latest quay.io/fetchit/fetchit:latest
+
+      - name: set values relating to the current env
+        run: | 
+          cd ci-rollback
+          sed -i 's|  url: http://github.com/containers/fetchit|  url: http://github.com/${{ github.repository }}|g' ./examples/ci-rollback.yaml
+          sed -i 's|   branch: ci-rollback|  branch: "{{ github.ref }}"|g' ./examples/ci-rollback.yaml
+      - name: Start fetchit
+        run: sudo podman run -d --name fetchit -v fetchit-volume:/opt -v /home/runner/work/fetchit/fetchit/ci-rollback/examples/ci-rollback.yaml:/opt/mount/config.yaml -v /run/podman/podman.sock:/run/podman/podman.sock --security-opt label=disable quay.io/fetchit/fetchit-amd:latest
+
+      - name: identify rollback1 container
+        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done'
+
+      - name: Logs
+        if: always()
+        run: sudo podman logs fetchit
+
+      - name: commit working
+        run: |
+          cd ci-rollback
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          ls
+          cp ./examples/rollback/1-working.yaml ./examples/ci-rollback/1-working.yaml
+          cp ./examples/rollback/2-broken.yaml ./examples/ci-rollback/2-broken.yaml
+          git add ./examples
+          git commit -m "add 1-working and 2-broken" -a
+          git push -f
+
+      - name: Wait for 2 pods
+        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 2 ]; do sudo podman ps; c=$(sudo podman ps | grep rollback | wc -l); done'
+
+      - name: Check rollback succeeded
+        run: timeout 150 bash -c -- 'c=0 ; until [ $c -eq 1 ]; do c=$(sudo podman ps | grep rollback | wc -l); done'
+
   multi-engine-validate:
     runs-on: ubuntu-latest
     needs: [ build , pull-and-archive ]

examples/ci-rollback.yaml

@@ -0,0 +1,10 @@
+targetConfigs:
+- name: fetchit
+  url: http://github.com/containers/fetchit
+  raw:
+  - name: raw
+    targetPath: examples/ci-rollback
+    schedule: "*/1 * * * *"
+  rollback: true
+  trackBadCommits: true
+  branch: ci-rollback

examples/raw/cap.yaml

@@ -1,14 +1,11 @@
-{
-    "Image":"docker.io/mmumshad/simple-webapp-color:latest",
-    "Name": "cap2",
-    "Env": {"APP_COLOR": "blue", "tree": "trunk"},
-    "Mounts": [],
-    "Volumes": [],
-    "CapDrop": ["all"],
-    "Ports": [{
-        "host_ip":        "",
-        "container_port": 8080,
-        "host_port":      9090,
-        "range":         0,
-        "protocol":      ""}]
-}
+Image: docker.io/mmumshad/simple-webapp-color:latest
+Name: cap2
+Env: 
+  APP_COLOR": blue
+  tree: trunk
+CapDrop: 
+  - all
+Ports: 
+  - container_port: 8080
+    host_port:      9091
+    range:         0

examples/rollback/1-working.yaml

@@ -0,0 +1,9 @@
+Image: "docker.io/mmumshad/simple-webapp-color:latest"
+Name: "rollback2"
+Env:
+  APP_COLOR: "pink"
+  tree: "trunk"
+Ports:
+  - container_port: 8080
+    host_port: 8081
+    range: 0

examples/rollback/2-broken.yaml

@@ -0,0 +1,10 @@
+# Typo below
+Image: "docker.io/mmumshad/simple-webapp-clor:latest"
+Name: "rollback3"
+Env:
+  APP_COLOR: "pink"
+  tree: "trunk"
+Ports:
+  - container_port: 8080
+    host_port: 8080
+    range: 0

examples/rollback/working.yaml

@@ -0,0 +1,9 @@
+Image: "docker.io/mmumshad/simple-webapp-color:latest"
+Name: "rollback1"
+Env:
+  APP_COLOR: "pink"
+  tree: "trunk"
+Ports:
+  - container_port: 8080
+    host_port: 8080
+    range: 0

go.mod

@@ -19,6 +19,7 @@ require (
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.23.5
 	k8s.io/apimachinery v0.23.5
+	k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -250,7 +251,6 @@ require (
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/klog/v2 v2.60.1-0.20220317184644-43cc75f9ae89 // indirect
 	k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect
 	sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect

pkg/engine/apply.go

@@ -153,6 +153,31 @@ func certHexFingerprint(cert *x509.Certificate) string {
 	return hex.EncodeToString(fpr[:])
 }
 
+func checkout(target *Target, hash plumbing.Hash) error {
+	if hash == plumbing.ZeroHash {
+		return nil
+	}
+
+	directory := getDirectory(target)
+
+	repo, err := git.PlainOpen(directory)
+	if err != nil {
+		return utils.WrapErr(err, "Error opening repository: %s", directory)
+	}
+
+	wt, err := repo.Worktree()
+	if err != nil {
+		return utils.WrapErr(err, "Error getting reference to worktree for repository", directory)
+	}
+
+	err = wt.Checkout(&git.CheckoutOptions{Hash: hash})
+	if err != nil {
+		return utils.WrapErr(err, "Error checking out %s on branch %s", hash, target.branch)
+	}
+
+	return nil
+}
+
 func getCurrent(target *Target, methodType, methodName string) (plumbing.Hash, error) {
 	directory := getDirectory(target)
 	tagName := fmt.Sprintf("current-%s-%s", methodType, methodName)

pkg/engine/common.go

@@ -7,8 +7,10 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/containers/fetchit/pkg/engine/utils"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/plumbing/object"
+	"k8s.io/klog/v2"
 )
 
 type CommonMethod struct {
@@ -70,6 +72,11 @@ func getDirectory(target *Target) string {
 	return filepath.Base(trimDir)
 }
 
+type empty struct {
+}
+
+var BadCommitList map[string]map[string]map[plumbing.Hash]empty = make(map[string]map[string]map[plumbing.Hash]empty)
+
 func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[]string) error {
 	directory := getDirectory(target)
 	if target.disconnected {
@@ -79,6 +86,7 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
 			localDevicePull(directory, target.device, "", false)
 		}
 	}
+
 	latest, err := getLatest(target)
 	if err != nil {
 		return fmt.Errorf("Failed to get latest commit: %v", err)
@@ -89,10 +97,43 @@ func currentToLatest(ctx, conn context.Context, m Method, target *Target, tag *[
 		return fmt.Errorf("Failed to get current commit: %v", err)
 	}
 
+	if target.rollback && target.trackBadCommits {
+		if _, ok := BadCommitList[directory]; !ok {
+			BadCommitList[directory] = make(map[string]map[plumbing.Hash]empty)
+		}
+
+		if _, ok := BadCommitList[directory][m.GetKind()]; !ok {
+			BadCommitList[directory][m.GetKind()] = make(map[plumbing.Hash]empty)
+		}
+
+		if _, ok := BadCommitList[directory][m.GetKind()][latest]; ok {
+			klog.Infof("No changes applied to target %s this run, %s currently at %s", directory, m.GetKind(), current)
+			return nil
+		}
+	}
+
 	if latest != current {
-		if err := m.Apply(ctx, conn, current, latest, tag); err != nil {
-			return fmt.Errorf("Failed to apply changes: %v", err)
+		if err = m.Apply(ctx, conn, current, latest, tag); err != nil {
+			if target.rollback {
+				// Roll back automatically
+				klog.Errorf("Failed to apply changes, rolling back to %v: %v", current, err)
+				if err = checkout(target, current); err != nil {
+					return utils.WrapErr(err, "Failed to checkout %s", current)
+				}
+				if err = m.Apply(ctx, conn, latest, current, tag); err != nil {
+					// Roll back failed
+					return fmt.Errorf("Roll back failed, state between %s and %s: %v", current, latest, err)
+				}
+				if target.trackBadCommits {
+					BadCommitList[directory][m.GetKind()][latest] = empty{}
+				}
+				return fmt.Errorf("Rolled back to %v: %v", current, err)
+			} else {
+				return fmt.Errorf("Failed to apply changes from %v to %v: %v", current, latest, err)
+			}
+
 		}
+
 		updateCurrent(ctx, target, latest, m.GetKind(), m.GetName())
 		logger.Infof("Moved %s from %s to %s for git target %s", m.GetName(), current.String()[:hashReportLen], latest, target.url)
 	} else {

pkg/engine/types.go

@@ -36,6 +36,8 @@ type TargetConfig struct {
 	Device            string             `mapstructure:"device"`
 	Disconnected      bool               `mapstructure:"disconnected"`
 	VerifyCommitsInfo *VerifyCommitsInfo `mapstructure:"verifyCommitsInfo"`
+	TrackBadCommits   bool               `mapstructure:"trackBadCommits"`
+	Rollback          bool               `mapstructure:"rollback"`
 	Branch            string             `mapstructure:"branch"`
 	Ansible           []*Ansible         `mapstructure:"ansible"`
 	FileTransfer      []*FileTransfer    `mapstructure:"filetransfer"`
@@ -58,6 +60,8 @@ type Target struct {
 	disconnected    bool
 	gitsignVerify   bool
 	gitsignRekorURL string
+	trackBadCommits bool
+	rollback        bool
 }
 
 type SchedInfo struct {