add compile_registry_state command

Author: efd6

docs/howto/script_testing.md

@@ -49,6 +49,7 @@ a stack, starting agents and services and validating results.
 - agent commands:
   - `install_agent [-profile <profile>] [-timeout <duration>] [-container_name <container_name_label>] [-network_name <network_name_label>]`: install an Elastic Agent policy, setting the environment variables named in the container_name and network_name arguments
   - `uninstall_agent [-profile <profile>] [-timeout <duration>]`: remove an installed Elastic Agent policy
+  - `compile_registry_state [-start <first_id_to_use>] <path_to_registry_log>`: compile a Filebeat registry log.json file into a registry state and print it to stdout
 
 - package commands:
   - `add_package [-profile <profile>] [-timeout <duration>]`: add the current package's assets

internal/testrunner/script/agents.go

@@ -6,10 +6,12 @@ package script
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"flag"
 	"fmt"
-	"path/filepath"
+	"io"
+	"os"
 	"time"
 
 	"github.com/rogpeppe/go-internal/testscript"
@@ -234,3 +236,83 @@ func deletePolicies(ctx context.Context, cli *kibana.Client, a *installedAgent)
 	}
 	return errors.Join(errs...)
 }
+
+func compileRegistryState(ts *testscript.TestScript, neg bool, args []string) {
+	if neg {
+		ts.Fatalf("unsupported: ! get_registry_state")
+	}
+	clearStdStreams(ts)
+	flg := flag.NewFlagSet("uninstall", flag.ContinueOnError)
+	first := flg.Int64("start", 0, "first registry log operation ID to use")
+	pretty := flg.Bool("pretty", false, "pretty print the registry")
+	ts.Check(flg.Parse(args))
+	if flg.NArg() != 1 {
+		ts.Fatalf("usage: get_registry_state [-start <first_id_to_use>] <path_to_registry_log>")
+	}
+	var choose func(int64) bool
+	if *first != 0 {
+		choose = func(i int64) bool {
+			return i >= *first
+		}
+	}
+	f, err := os.Open(ts.MkAbs(flg.Arg(0)))
+	defer f.Close()
+	ts.Check(decoratedWith("opening registry log", err))
+
+	s := make(map[string]any)
+	ts.Check(decoratedWith("compiling state", compileStateInto(s, f, choose)))
+	var msg []byte
+	if *pretty {
+		msg, err = json.MarshalIndent(s, "", "  ")
+	} else {
+		msg, err = json.Marshal(s)
+	}
+	ts.Check(decoratedWith("marshaling registry", err))
+	fmt.Fprintf(ts.Stdout(), "%s\n", msg)
+}
+
+func compileStateInto(dst map[string]any, r io.Reader, choose func(int64) bool) error {
+	type action struct {
+		Operation string `json:"op"`
+		ID        int64  `json:"id"`
+	}
+
+	type delta struct {
+		Key string         `json:"K"`
+		Val map[string]any `json:"V"`
+	}
+
+	dec := json.NewDecoder(r)
+	dec.DisallowUnknownFields()
+	for {
+		var (
+			a action
+			d delta
+		)
+		err := dec.Decode(&a)
+		if err != nil {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+		err = dec.Decode(&d)
+		if err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if choose != nil && !choose(a.ID) {
+			continue
+		}
+		switch a.Operation {
+		case "set":
+			dst[d.Key] = d.Val
+		case "remove":
+			delete(dst, d.Key)
+		default:
+			return fmt.Errorf("unknown operation: %q", a.Operation)
+		}
+	}
+}

internal/testrunner/script/script.go

@@ -211,6 +211,7 @@ func Run(dst io.Writer, cmd *cobra.Command, args []string) error {
 				"dump_logs":              dumpLogs,
 				"match_file":             match,
 				"get_policy":             getPolicyCommand,
+				"compile_registry_state":     compileRegistryState,
 			},
 			Setup: func(e *testscript.Env) error {
 				e.Setenv("PROFILE", config.CurrentProfile())

test/packages/other/with_script/data_stream/first/_dev/test/scripts/registry.txt

@@ -0,0 +1,113 @@
+compile_registry_state -pretty keys_log.json
+cmp stdout want_keys_log_0.json
+
+compile_registry_state -pretty -start=3 keys_log.json
+cmp stdout want_keys_log_3.json
+
+compile_registry_state -pretty key_log.json
+cmp stdout want_key_log.json
+
+compile_registry_state -pretty keys_removed_log.json
+cmp stdout want_keys_removed_log.json
+
+-- keys_log.json --
+{"op":"set", "id": 1}
+{"K":"key0","V":{"a":0}}
+{"op":"set", "id": 2}
+{"K":"key1","V":{"a":1}}
+{"op":"set", "id": 3}
+{"K":"key2","V":{"a":2}}
+{"op":"set", "id": 4}
+{"K":"key3","V":{"a":3}}
+{"op":"set", "id": 5}
+{"K":"key4","V":{"a":4}}
+{"op":"set", "id": 6}
+{"K":"key5","V":{"a":5}}
+-- want_keys_log_0.json --
+{
+  "key0": {
+    "a": 0
+  },
+  "key1": {
+    "a": 1
+  },
+  "key2": {
+    "a": 2
+  },
+  "key3": {
+    "a": 3
+  },
+  "key4": {
+    "a": 4
+  },
+  "key5": {
+    "a": 5
+  }
+}
+-- want_keys_log_3.json --
+{
+  "key2": {
+    "a": 2
+  },
+  "key3": {
+    "a": 3
+  },
+  "key4": {
+    "a": 4
+  },
+  "key5": {
+    "a": 5
+  }
+}
+-- key_log.json --
+{"op":"set", "id": 1}
+{"K":"key0","V":{"a":0}}
+{"op":"set", "id": 2}
+{"K":"key0","V":{"a":1}}
+{"op":"set", "id": 3}
+{"K":"key0","V":{"a":2}}
+{"op":"set", "id": 4}
+{"K":"key0","V":{"a":3}}
+{"op":"set", "id": 5}
+{"K":"key0","V":{"a":4}}
+{"op":"set", "id": 6}
+{"K":"key0","V":{"a":5}}
+-- want_key_log.json --
+{
+  "key0": {
+    "a": 5
+  }
+}
+-- keys_removed_log.json --
+{"op":"set", "id": 1}
+{"K":"key0","V":{"a":0}}
+{"op":"set", "id": 2}
+{"K":"key1","V":{"a":1}}
+{"op":"set", "id": 3}
+{"K":"key2","V":{"a":2}}
+{"op":"set", "id": 4}
+{"K":"key3","V":{"a":3}}
+{"op":"set", "id": 5}
+{"K":"key4","V":{"a":4}}
+{"op":"set", "id": 6}
+{"K":"key5","V":{"a":5}}
+{"op":"remove", "id": 7}
+{"K":"key3"}
+-- want_keys_removed_log.json --
+{
+  "key0": {
+    "a": 0
+  },
+  "key1": {
+    "a": 1
+  },
+  "key2": {
+    "a": 2
+  },
+  "key4": {
+    "a": 4
+  },
+  "key5": {
+    "a": 5
+  }
+}