diff --git a/api/datareading.go b/api/datareading.go index 54637f3c..75e7dee3 100644 --- a/api/datareading.go +++ b/api/datareading.go @@ -1,8 +1,12 @@ package api import ( + "bytes" "encoding/json" "time" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/version" ) // DataReadingsPost is the payload in the upload request. @@ -28,8 +32,8 @@ type DataReading struct { type GatheredResource struct { // Resource is a reference to a k8s object that was found by the informer // should be of type unstructured.Unstructured, raw Object - Resource interface{} - DeletedAt Time + Resource interface{} `json:"resource"` + DeletedAt Time `json:"deleted_at,omitempty"` } func (v GatheredResource) MarshalJSON() ([]byte, error) { @@ -48,3 +52,32 @@ func (v GatheredResource) MarshalJSON() ([]byte, error) { return json.Marshal(data) } + +func (v *GatheredResource) UnmarshalJSON(data []byte) error { + var tmpResource struct { + Resource *unstructured.Unstructured `json:"resource"` + DeletedAt Time `json:"deleted_at,omitempty"` + } + + d := json.NewDecoder(bytes.NewReader(data)) + d.DisallowUnknownFields() + + if err := d.Decode(&tmpResource); err != nil { + return err + } + v.Resource = tmpResource.Resource + v.DeletedAt = tmpResource.DeletedAt + return nil +} + +// DynamicData is the DataReading.Data returned by the k8s.DataGathererDynamic +// gatherer +type DynamicData struct { + Items []*GatheredResource `json:"items"` +} + +// DiscoveryData is the DataReading.Data returned by the k8s.ConfigDiscovery +// gatherer +type DiscoveryData struct { + ServerVersion *version.Info `json:"server_version"` +} diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 6995c548..13d4401f 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -321,6 +321,14 @@ func gatherAndOutputData(ctx context.Context, eventf Eventf, config CombinedConf var readings []*api.DataReading if config.InputPath != "" { + // TODO(wallrj): The datareadings read from disk can not yet be pushed + // to the CyberArk Discovery and Context API. Why? Because they have + // simple data types such as map[string]interface{}. In contrast, the + // data from data gatherers can be cast to rich types like DynamicData + // or DiscoveryData The CyberArk dataupload client requires the data to + // have rich types to convert it to the Discovery and Context snapshots + // format. Consider refactoring testutil.ParseDataReadings so that it + // can be used here. log.V(logs.Debug).Info("Reading data from local file", "inputPath", config.InputPath) data, err := os.ReadFile(config.InputPath) if err != nil { diff --git a/pkg/client/client_cyberark.go b/pkg/client/client_cyberark.go deleted file mode 100644 index af26e96c..00000000 --- a/pkg/client/client_cyberark.go +++ /dev/null @@ -1,9 +0,0 @@ -package client - -import ( - "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" -) - -type CyberArkClient = dataupload.CyberArkClient - -var NewCyberArkClient = dataupload.NewCyberArkClient diff --git a/pkg/datagatherer/k8s/discovery.go b/pkg/datagatherer/k8s/discovery.go index 586622d6..340bcbda 100644 --- a/pkg/datagatherer/k8s/discovery.go +++ b/pkg/datagatherer/k8s/discovery.go @@ -6,6 +6,7 @@ import ( "k8s.io/client-go/discovery" + "github.com/jetstack/preflight/api" "github.com/jetstack/preflight/pkg/datagatherer" ) @@ -59,15 +60,12 @@ func (g *DataGathererDiscovery) WaitForCacheSync(ctx context.Context) error { // Fetch will fetch discovery data from the apiserver, or return an error func (g *DataGathererDiscovery) Fetch() (interface{}, int, error) { - data, err := g.cl.ServerVersion() + serverVersion, err := g.cl.ServerVersion() if err != nil { return nil, -1, fmt.Errorf("failed to get server version: %v", err) } - response := map[string]interface{}{ - // data has type Info: https://godoc.org/k8s.io/apimachinery/pkg/version#Info - "server_version": data, - } - - return response, len(response), nil + return &api.DiscoveryData{ + ServerVersion: serverVersion, + }, 1, nil } diff --git a/pkg/datagatherer/k8s/dynamic.go b/pkg/datagatherer/k8s/dynamic.go index b0e1dedf..9dbffe88 100644 --- a/pkg/datagatherer/k8s/dynamic.go +++ b/pkg/datagatherer/k8s/dynamic.go @@ -314,7 +314,6 @@ func (g *DataGathererDynamic) Fetch() (interface{}, int, error) { return nil, -1, fmt.Errorf("resource type must be specified") } - var list = map[string]interface{}{} var items = []*api.GatheredResource{} fetchNamespaces := g.namespaces @@ -344,10 +343,9 @@ func (g *DataGathererDynamic) Fetch() (interface{}, int, error) { return nil, -1, err } - // add gathered resources to items - list["items"] = items - - return list, len(items), nil + return &api.DynamicData{ + Items: items, + }, len(items), nil } func redactList(list []*api.GatheredResource, excludeAnnotKeys, excludeLabelKeys []*regexp.Regexp) error { diff --git a/pkg/datagatherer/k8s/dynamic_test.go b/pkg/datagatherer/k8s/dynamic_test.go index 072c4c1c..525c8892 100644 --- a/pkg/datagatherer/k8s/dynamic_test.go +++ b/pkg/datagatherer/k8s/dynamic_test.go @@ -730,15 +730,12 @@ func TestDynamicGatherer_Fetch(t *testing.T) { } if tc.expected != nil { - items, ok := res.(map[string]interface{}) + data, ok := res.(*api.DynamicData) if !ok { - t.Errorf("expected result be an map[string]interface{} but wasn't") + t.Errorf("expected result be *api.DynamicData but wasn't") } - list, ok := items["items"].([]*api.GatheredResource) - if !ok { - t.Errorf("expected result be an []*api.GatheredResource but wasn't") - } + list := data.Items // sorting list of results by name sortGatheredResources(list) // sorting list of expected results by name @@ -1045,10 +1042,9 @@ func TestDynamicGathererNativeResources_Fetch(t *testing.T) { } if tc.expected != nil { - res, ok := rawRes.(map[string]interface{}) - require.Truef(t, ok, "expected result be an map[string]interface{} but wasn't") - actual := res["items"].([]*api.GatheredResource) - require.Truef(t, ok, "expected result be an []*api.GatheredResource but wasn't") + res, ok := rawRes.(*api.DynamicData) + require.Truef(t, ok, "expected result be an *api.DynamicData but wasn't") + actual := res.Items // sorting list of results by name sortGatheredResources(actual) diff --git a/pkg/datagatherer/k8s/fieldfilter.go b/pkg/datagatherer/k8s/fieldfilter.go index ed39acb3..1bddd387 100644 --- a/pkg/datagatherer/k8s/fieldfilter.go +++ b/pkg/datagatherer/k8s/fieldfilter.go @@ -16,6 +16,9 @@ var SecretSelectedFields = []FieldPath{ {"metadata", "ownerReferences"}, {"metadata", "selfLink"}, {"metadata", "uid"}, + {"metadata", "creationTimestamp"}, + {"metadata", "deletionTimestamp"}, + {"metadata", "resourceVersion"}, {"type"}, {"data", "tls.crt"}, diff --git a/pkg/internal/cyberark/dataupload/dataupload.go b/pkg/internal/cyberark/dataupload/dataupload.go index d8835e3b..014aa85b 100644 --- a/pkg/internal/cyberark/dataupload/dataupload.go +++ b/pkg/internal/cyberark/dataupload/dataupload.go @@ -58,14 +58,19 @@ func NewCyberArkClient(trustedCAs *x509.CertPool, baseURL string, authenticateRe // PostDataReadingsWithOptions PUTs the supplied payload to an [AWS presigned URL] which it obtains via the CyberArk inventory API. // // [AWS presigned URL]: https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html -func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payload api.DataReadingsPost, opts Options) error { +func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, readings []*api.DataReading, opts Options) error { if opts.ClusterName == "" { return fmt.Errorf("programmer mistake: the cluster name (aka `cluster_id` in the config file) cannot be left empty") } + snapshot, err := convertDataReadingsToCyberarkSnapshot(readings) + if err != nil { + return fmt.Errorf("while converting datareadings to Cyberark snapshot format: %s", err) + } + encodedBody := &bytes.Buffer{} checksum := sha3.New256() - if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(payload); err != nil { + if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(snapshot); err != nil { return err } diff --git a/pkg/internal/cyberark/dataupload/dataupload_test.go b/pkg/internal/cyberark/dataupload/dataupload_test.go index cadb296a..4639029f 100644 --- a/pkg/internal/cyberark/dataupload/dataupload_test.go +++ b/pkg/internal/cyberark/dataupload/dataupload_test.go @@ -3,6 +3,7 @@ package dataupload_test import ( "crypto/x509" "encoding/pem" + "errors" "fmt" "net/http" "os" @@ -17,28 +18,23 @@ import ( "github.com/jetstack/preflight/pkg/internal/cyberark/dataupload" "github.com/jetstack/preflight/pkg/internal/cyberark/identity" "github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery" + "github.com/jetstack/preflight/pkg/testutil" _ "k8s.io/klog/v2/ktesting/init" ) -func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { +func TestCyberArkClient_PostDataReadingsWithOptions_MockAPI(t *testing.T) { fakeTime := time.Unix(123, 0) - defaultPayload := api.DataReadingsPost{ - AgentMetadata: &api.AgentMetadata{ - Version: "test-version", - ClusterID: "test", - }, - DataGatherTime: fakeTime, - DataReadings: []*api.DataReading{ - { - ClusterID: "success-cluster-id", - DataGatherer: "test-gatherer", - Timestamp: api.Time{Time: fakeTime}, - Data: map[string]interface{}{"test": "data"}, - SchemaVersion: "v1", - }, + defaultDataReadings := []*api.DataReading{ + { + ClusterID: "success-cluster-id", + DataGatherer: "test-gatherer", + Timestamp: api.Time{Time: fakeTime}, + Data: map[string]interface{}{"test": "data"}, + SchemaVersion: "v1", }, } + defaultOpts := dataupload.Options{ ClusterName: "success-cluster-id", } @@ -52,14 +48,14 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { tests := []struct { name string - payload api.DataReadingsPost + readings []*api.DataReading authenticate func(req *http.Request) error opts dataupload.Options requireFn func(t *testing.T, err error) }{ { name: "successful upload", - payload: defaultPayload, + readings: defaultDataReadings, opts: defaultOpts, authenticate: setToken("success-token"), requireFn: func(t *testing.T, err error) { @@ -68,7 +64,7 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { }, { name: "error when cluster name is empty", - payload: defaultPayload, + readings: defaultDataReadings, opts: dataupload.Options{ClusterName: ""}, authenticate: setToken("success-token"), requireFn: func(t *testing.T, err error) { @@ -77,16 +73,27 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { }, { name: "error when bearer token is incorrect", - payload: defaultPayload, + readings: defaultDataReadings, opts: defaultOpts, authenticate: setToken("fail-token"), requireFn: func(t *testing.T, err error) { require.ErrorContains(t, err, "while retrieving snapshot upload URL: received response with status code 500: should authenticate using the correct bearer token") }, }, + { + name: "error contains authenticate error", + readings: defaultDataReadings, + opts: defaultOpts, + authenticate: func(_ *http.Request) error { + return errors.New("simulated-authenticate-error") + }, + requireFn: func(t *testing.T, err error) { + require.ErrorContains(t, err, "while retrieving snapshot upload URL: failed to authenticate request: simulated-authenticate-error") + }, + }, { name: "invalid JSON from server (RetrievePresignedUploadURL step)", - payload: defaultPayload, + readings: defaultDataReadings, opts: dataupload.Options{ClusterName: "invalid-json-retrieve-presigned"}, authenticate: setToken("success-token"), requireFn: func(t *testing.T, err error) { @@ -95,7 +102,7 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { }, { name: "500 from server (RetrievePresignedUploadURL step)", - payload: defaultPayload, + readings: defaultDataReadings, opts: dataupload.Options{ClusterName: "invalid-response-post-data"}, authenticate: setToken("success-token"), requireFn: func(t *testing.T, err error) { @@ -106,6 +113,9 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { + logger := ktesting.NewLogger(t, ktesting.DefaultConfig) + ctx := klog.NewContext(t.Context(), logger) + server := dataupload.MockDataUploadServer() defer server.Close() @@ -118,13 +128,13 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { cyberArkClient, err := dataupload.NewCyberArkClient(certPool, server.Server.URL, tc.authenticate) require.NoError(t, err) - err = cyberArkClient.PostDataReadingsWithOptions(t.Context(), tc.payload, tc.opts) + err = cyberArkClient.PostDataReadingsWithOptions(ctx, tc.readings, tc.opts) tc.requireFn(t, err) }) } } -// TestPostDataReadingsWithOptionsWithRealAPI demonstrates that the dataupload code works with the real inventory API. +// TestCyberArkClient_PostDataReadingsWithOptions_RealAPI demonstrates that the dataupload code works with the real inventory API. // An API token is obtained by authenticating with the ARK_USERNAME and ARK_SECRET from the environment. // ARK_SUBDOMAIN should be your tenant subdomain. // ARK_PLATFORM_DOMAIN should be either integration-cyberark.cloud or cyberark.cloud @@ -132,8 +142,8 @@ func TestCyberArkClient_PostDataReadingsWithOptions(t *testing.T) { // To enable verbose request logging: // // go test ./pkg/internal/cyberark/dataupload/... \ -// -v -count 1 -run TestPostDataReadingsWithOptionsWithRealAPI -args -testing.v 6 -func TestPostDataReadingsWithOptionsWithRealAPI(t *testing.T) { +// -v -count 1 -run TestCyberArkClient_PostDataReadingsWithOptions_RealAPI -args -testing.v 6 +func TestCyberArkClient_PostDataReadingsWithOptions_RealAPI(t *testing.T) { platformDomain := os.Getenv("ARK_PLATFORM_DOMAIN") subdomain := os.Getenv("ARK_SUBDOMAIN") username := os.Getenv("ARK_USERNAME") @@ -172,8 +182,13 @@ func TestPostDataReadingsWithOptionsWithRealAPI(t *testing.T) { cyberArkClient, err := dataupload.NewCyberArkClient(nil, serviceURL, identityClient.AuthenticateRequest) require.NoError(t, err) - err = cyberArkClient.PostDataReadingsWithOptions(ctx, api.DataReadingsPost{}, dataupload.Options{ - ClusterName: "bb068932-c80d-460d-88df-34bc7f3f3297", - }) + dataReadings := testutil.ParseDataReadings(t, testutil.ReadGZIP(t, "testdata/example-1/datareadings.json.gz")) + err = cyberArkClient.PostDataReadingsWithOptions( + ctx, + dataReadings, + dataupload.Options{ + ClusterName: "bb068932-c80d-460d-88df-34bc7f3f3297", + }, + ) require.NoError(t, err) } diff --git a/pkg/internal/cyberark/dataupload/mock.go b/pkg/internal/cyberark/dataupload/mock.go index f8a2530b..5887253c 100644 --- a/pkg/internal/cyberark/dataupload/mock.go +++ b/pkg/internal/cyberark/dataupload/mock.go @@ -1,6 +1,7 @@ package dataupload import ( + "bytes" "crypto/sha3" "encoding/hex" "encoding/json" @@ -41,10 +42,7 @@ func (mds *mockDataUploadServer) ServeHTTP(w http.ResponseWriter, r *http.Reques mds.handlePresignedUpload(w, r) return case "/presigned-upload": - mds.handleUpload(w, r, false) - return - case "/presigned-upload-invalid-json": - mds.handleUpload(w, r, false) + mds.handleUpload(w, r) return default: w.WriteHeader(http.StatusNotFound) @@ -54,7 +52,10 @@ func (mds *mockDataUploadServer) ServeHTTP(w http.ResponseWriter, r *http.Reques func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) - _, _ = w.Write([]byte(`{"message":"method not allowed"}`)) + _, err := w.Write([]byte(`{"message":"method not allowed"}`)) + if err != nil { + panic(err) + } return } @@ -94,7 +95,10 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r if req.ClusterID == "invalid-json-retrieve-presigned" { w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"url":`)) // invalid JSON + _, err := w.Write([]byte(`{"url":`)) // invalid JSON + if err != nil { + panic(err) + } return } @@ -118,10 +122,13 @@ func (mds *mockDataUploadServer) handlePresignedUpload(w http.ResponseWriter, r }{presignedURL}) } -func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Request, invalidJSON bool) { +func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPut { w.WriteHeader(http.StatusMethodNotAllowed) - _, _ = w.Write([]byte(`{"message":"method not allowed"}`)) + _, err := w.Write([]byte(`{"message":"method not allowed"}`)) + if err != nil { + panic(err) + } return } @@ -130,21 +137,26 @@ func (mds *mockDataUploadServer) handleUpload(w http.ResponseWriter, r *http.Req return } - if invalidJSON { - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"url":`)) // invalid JSON - return + body, err := io.ReadAll(r.Body) + if err != nil { + panic(err) } checksum := sha3.New256() - _, _ = io.Copy(checksum, r.Body) - + _, err = checksum.Write(body) + if err != nil { + panic(err) + } if r.URL.Query().Get("checksum") != hex.EncodeToString(checksum.Sum(nil)) { http.Error(w, "checksum is invalid", http.StatusInternalServerError) } + var snapshot snapshot + d := json.NewDecoder(bytes.NewBuffer(body)) + d.DisallowUnknownFields() + if err := d.Decode(&snapshot); err != nil { + panic(err) + } + w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{"success":true}`)) } diff --git a/pkg/internal/cyberark/dataupload/snapshot.go b/pkg/internal/cyberark/dataupload/snapshot.go new file mode 100644 index 00000000..41e2c9e5 --- /dev/null +++ b/pkg/internal/cyberark/dataupload/snapshot.go @@ -0,0 +1,102 @@ +package dataupload + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/jetstack/preflight/api" + "github.com/jetstack/preflight/pkg/version" +) + +type resourceData map[string][]*unstructured.Unstructured + +// snapshot is the JSON that the CyberArk Discovery and Context API expects to +// be uploaded to the AWS presigned URL. +type snapshot struct { + AgentVersion string `json:"agent_version"` + ClusterID string `json:"cluster_id"` + K8SVersion string `json:"k8s_version"` + Secrets []*unstructured.Unstructured `json:"secrets"` + ServiceAccounts []*unstructured.Unstructured `json:"service_accounts"` + Roles []*unstructured.Unstructured `json:"roles"` + RoleBindings []*unstructured.Unstructured `json:"role_bindings"` +} + +// The names of Datagatherers which have the data to populate the Cyberark +// Snapshot mapped to the key in the Cyberark snapshot. +var gathererNameToResourceDataKeyMap = map[string]string{ + "ark/secrets": "secrets", + "ark/serviceaccounts": "serviceaccounts", + "ark/roles": "roles", + "ark/clusterroles": "roles", + "ark/rolebindings": "rolebindings", + "ark/clusterrolebindings": "rolebindings", +} + +// extractResourceListFromReading converts the opaque data from a DynamicData +// data reading to Unstructured resources, to allow access to the metadata and +// other kubernetes API fields. +func extractResourceListFromReading(reading *api.DataReading) ([]*unstructured.Unstructured, error) { + data, ok := reading.Data.(*api.DynamicData) + if !ok { + return nil, fmt.Errorf("failed to convert data: %s", reading.DataGatherer) + } + items := data.Items + resources := make([]*unstructured.Unstructured, len(items)) + for i, item := range items { + if resource, ok := item.Resource.(*unstructured.Unstructured); ok { + resources[i] = resource + } else { + return nil, fmt.Errorf("failed to convert resource: %#v", item) + } + } + return resources, nil +} + +// extractServerVersionFromReading converts the opaque data from a DiscoveryData +// data reding to allow access to the Kubernetes version fields within. +func extractServerVersionFromReading(reading *api.DataReading) (string, error) { + data, ok := reading.Data.(*api.DiscoveryData) + if !ok { + return "", fmt.Errorf("failed to convert data: %s", reading.DataGatherer) + } + if data.ServerVersion == nil { + return "unknown", nil + } + return data.ServerVersion.GitVersion, nil +} + +// convertDataReadingsToCyberarkSnapshot converts DataReadings to the Cyberark +// Snapshot format. +func convertDataReadingsToCyberarkSnapshot( + readings []*api.DataReading, +) (*snapshot, error) { + k8sVersion := "" + resourceData := resourceData{} + for _, reading := range readings { + if reading.DataGatherer == "ark/discovery" { + var err error + k8sVersion, err = extractServerVersionFromReading(reading) + if err != nil { + return nil, fmt.Errorf("while extracting server version from data-reading: %s", err) + } + } + if key, found := gathererNameToResourceDataKeyMap[reading.DataGatherer]; found { + resources, err := extractResourceListFromReading(reading) + if err != nil { + return nil, fmt.Errorf("while extracting resource list from data-reading: %s", err) + } + resourceData[key] = append(resourceData[key], resources...) + } + } + + return &snapshot{ + AgentVersion: version.PreflightVersion, + K8SVersion: k8sVersion, + Secrets: resourceData["secrets"], + ServiceAccounts: resourceData["serviceaccounts"], + Roles: resourceData["roles"], + RoleBindings: resourceData["rolebindings"], + }, nil +} diff --git a/pkg/internal/cyberark/dataupload/snapshot_test.go b/pkg/internal/cyberark/dataupload/snapshot_test.go new file mode 100644 index 00000000..f9fae63e --- /dev/null +++ b/pkg/internal/cyberark/dataupload/snapshot_test.go @@ -0,0 +1,29 @@ +package dataupload + +import ( + "encoding/json" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/jetstack/preflight/pkg/testutil" +) + +func TestConvertDataReadingsToCyberarkSnapshot(t *testing.T) { + dataReadings := testutil.ParseDataReadings(t, testutil.ReadGZIP(t, "testdata/example-1/datareadings.json.gz")) + snapshot, err := convertDataReadingsToCyberarkSnapshot(dataReadings) + require.NoError(t, err) + + actualSnapshotBytes, err := json.MarshalIndent(snapshot, "", " ") + require.NoError(t, err) + + goldenFilePath := "testdata/example-1/snapshot.json.gz" + if _, update := os.LookupEnv("UPDATE_GOLDEN_FILES"); update { + testutil.WriteGZIP(t, goldenFilePath, actualSnapshotBytes) + } else { + expectedSnapshotBytes := testutil.ReadGZIP(t, goldenFilePath) + assert.JSONEq(t, string(expectedSnapshotBytes), string(actualSnapshotBytes)) + } +} diff --git a/pkg/internal/cyberark/dataupload/testdata/example-1/README.md b/pkg/internal/cyberark/dataupload/testdata/example-1/README.md new file mode 100644 index 00000000..0ea38f86 --- /dev/null +++ b/pkg/internal/cyberark/dataupload/testdata/example-1/README.md @@ -0,0 +1,27 @@ +# README + +Data captured from a cert-manager E2E test cluster. + +```bash +cd cert-manager +make e2e-setup +``` + +```bash +cd jetstack-secure +go run . agent \ + --api-token not-used \ + --install-namespace venafi \ + --log-level 6 \ + --one-shot \ + --agent-config-file pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml \ + --output-path pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json +gzip pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json +``` + + +To recreate the golden output file: + +```bash +UPDATE_GOLDEN_FILES=true go test ./pkg/internal/cyberark/dataupload/... -run TestConvertDataReadingsToCyberarkSnapshot +``` diff --git a/pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml b/pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml new file mode 100644 index 00000000..27f4f38b --- /dev/null +++ b/pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml @@ -0,0 +1,54 @@ +cluster_id: example-cluster-id +organization_id: example-organization-id +data-gatherers: +# gather k8s apiserver version information +- kind: k8s-discovery + name: ark/discovery +- kind: k8s-dynamic + name: ark/serviceaccounts + config: + resource-type: + resource: serviceaccounts + version: v1 +- kind: k8s-dynamic + name: ark/secrets + config: + resource-type: + version: v1 + resource: secrets + field-selectors: + - type!=kubernetes.io/service-account-token + - type!=kubernetes.io/dockercfg + - type!=kubernetes.io/dockerconfigjson + - type!=kubernetes.io/basic-auth + - type!=kubernetes.io/ssh-auth + - type!=bootstrap.kubernetes.io/token + - type!=helm.sh/release.v1 +- kind: k8s-dynamic + name: ark/roles + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: roles +- kind: k8s-dynamic + name: ark/clusterroles + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: clusterroles +- kind: k8s-dynamic + name: ark/rolebindings + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: rolebindings +- kind: k8s-dynamic + name: ark/clusterrolebindings + config: + resource-type: + version: v1 + group: rbac.authorization.k8s.io + resource: clusterrolebindings diff --git a/pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json.gz b/pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json.gz new file mode 100644 index 00000000..27d1e4ab Binary files /dev/null and b/pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json.gz differ diff --git a/pkg/internal/cyberark/dataupload/testdata/example-1/snapshot.json.gz b/pkg/internal/cyberark/dataupload/testdata/example-1/snapshot.json.gz new file mode 100644 index 00000000..68ca44bb Binary files /dev/null and b/pkg/internal/cyberark/dataupload/testdata/example-1/snapshot.json.gz differ diff --git a/pkg/testutil/datareadings.go b/pkg/testutil/datareadings.go new file mode 100644 index 00000000..9b281325 --- /dev/null +++ b/pkg/testutil/datareadings.go @@ -0,0 +1,90 @@ +package testutil + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/jetstack/preflight/api" +) + +// ParseDataReadings decodes JSON encoded datareadings. +// It attempts to decode the data of each reading into a concrete type. +// It tries to decode the data as DynamicData and DiscoveryData and then gives +// up with a test failure. +// This function is useful for reading sample datareadings from disk for use in +// CyberArk dataupload client tests, which require the datareadings data to have +// rich types +// TODO(wallrj): Refactor this so that it can be used with the `agent +// --input-path` feature, to enable datareadings to be read from disk and pushed +// to CyberArk. +func ParseDataReadings(t *testing.T, data []byte) []*api.DataReading { + var dataReadings []*api.DataReading + + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + err := decoder.Decode(&dataReadings) + require.NoError(t, err) + + for _, reading := range dataReadings { + dataBytes, err := json.Marshal(reading.Data) + require.NoError(t, err) + in := bytes.NewReader(dataBytes) + d := json.NewDecoder(in) + d.DisallowUnknownFields() + + var dynamicGatherData api.DynamicData + if err := d.Decode(&dynamicGatherData); err == nil { + reading.Data = &dynamicGatherData + continue + } + + _, err = in.Seek(0, 0) + require.NoError(t, err) + + var discoveryData api.DiscoveryData + if err = d.Decode(&discoveryData); err == nil { + reading.Data = &discoveryData + continue + } + + require.Failf(t, "failed to parse reading", "reading: %#v", reading) + } + return dataReadings +} + +// ReadGZIP Reads the gzip file at path, and returns the decompressed bytes +func ReadGZIP(t *testing.T, path string) []byte { + f, err := os.Open(path) + require.NoError(t, err) + defer func() { require.NoError(t, f.Close()) }() + gzr, err := gzip.NewReader(f) + require.NoError(t, err) + defer func() { require.NoError(t, gzr.Close()) }() + bytes, err := io.ReadAll(gzr) + require.NoError(t, err) + return bytes +} + +// WriteGZIP writes gzips the data and writes it to path. +func WriteGZIP(t *testing.T, path string, data []byte) { + tmp, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*") + require.NoError(t, err) + gzw := gzip.NewWriter(tmp) + _, err = gzw.Write(data) + require.NoError(t, errors.Join( + err, + gzw.Flush(), + gzw.Close(), + tmp.Close(), + )) + err = os.Rename(tmp.Name(), path) + require.NoError(t, err) +}