diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 7c9cef90..a765f1eb 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -130,7 +130,7 @@ jobs: - name: Golang Test Helper if: ${{ needs.analyze-changes.outputs.go_helper_any_changed == 'true' }} run: | - go test -v ./test + go test -v ./test -timeout 30m - name: Upload KinD logs if: always() diff --git a/.gitignore b/.gitignore index d5a66362..f354e865 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ kubeconfig hosts.yml vendor/* -tls.crt -tls.key -public-key-cert.pem -**/kind-logs \ No newline at end of file +*.crt +*.key +*.pem +**/kind-logs diff --git a/go.mod b/go.mod index f63c4c96..b3e6f499 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module test +module e2eutils go 1.24.0 diff --git a/kubernetes-services/templates/argo-rollouts.yaml b/kubernetes-services/templates/argo-rollouts.yaml index fad10a96..d6ad5732 100644 --- a/kubernetes-services/templates/argo-rollouts.yaml +++ b/kubernetes-services/templates/argo-rollouts.yaml @@ -23,4 +23,4 @@ spec: prune: true selfHeal: true syncOptions: - - CreateNamespace=true \ No newline at end of file + - CreateNamespace=true diff --git a/kubernetes-services/templates/kargo.yaml b/kubernetes-services/templates/kargo.yaml deleted file mode 100644 index 5366ddce..00000000 --- a/kubernetes-services/templates/kargo.yaml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: argoproj.io/v1alpha1 -kind: Application -metadata: - name: kargo - namespace: argo-cd - finalizers: - - resources-finalizer.argocd.argoproj.io -spec: - destination: - server: https://kubernetes.default.svc - namespace: kargo - project: kubernetes - sources: - - chart: kargo - repoURL: ghcr.io/akuity/kargo-charts - targetRevision: 1.5.3 - helm: - values: | - api: - # hardcoded login credentials - # in production systems should use oidc! - adminAccount: - passwordHash: "$2a$12$/fHRdnXaUYBicfR0BsKh/.el6l4O/o.fEeGI7yyOjchEIfYj5Mh.K" - tokenSigningKey: AbAugOWkStfbwczR8wQooceM3 - syncPolicy: - automated: - prune: true - selfHeal: true - syncOptions: - - CreateNamespace=true \ No newline at end of file diff --git a/test/pkg/api/api.go b/pkg/api.go similarity index 96% rename from test/pkg/api/api.go rename to pkg/api.go index a0822246..7bd9786d 100644 --- a/test/pkg/api/api.go +++ b/pkg/api.go @@ -1,8 +1,9 @@ -package api +package e2eutils import ( "context" "errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,7 +13,6 @@ import ( "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" "sigs.k8s.io/e2e-framework/klient/k8s" - "test/test/pkg/test" ) func Apply(clientset kubernetes.Clientset, object runtime.Object) error { @@ -33,7 +33,7 @@ func Apply(clientset kubernetes.Clientset, object runtime.Object) error { _, err := createPersistentVolumeClaim(clientset, *object.(*corev1.PersistentVolumeClaim)) return err case *unstructured.Unstructured: - dynClient, err := test.GetDynClient() + dynClient, err := GetDynClient() if err != nil { return err } @@ -75,7 +75,7 @@ func createPersistentVolumeClaim(clientset kubernetes.Clientset, object corev1.P } func getResourceName(object unstructured.Unstructured) (string, error) { - discoveryClient, err := test.GetDiscoveryClient() + discoveryClient, err := GetDiscoveryClient() if err != nil { return "", errors.New("Failed to get discovery client " + err.Error()) } diff --git a/test/pkg/argo/application.go b/pkg/argo/application.go similarity index 100% rename from test/pkg/argo/application.go rename to pkg/argo/application.go diff --git a/test/pkg/argo/argo.go b/pkg/argo/argo.go similarity index 72% rename from test/pkg/argo/argo.go rename to pkg/argo/argo.go index c84e3284..a9690afa 100644 --- a/test/pkg/argo/argo.go +++ b/pkg/argo/argo.go @@ -5,12 +5,16 @@ import ( "io" "net/http" "os" - "sigs.k8s.io/yaml" "strconv" "strings" + + "sigs.k8s.io/yaml" ) -func GetArgoApplication(applicationYaml string) (Application, error) { +func GetArgoApplication(applicationYaml string) ( + Application, + error, +) { yamlFile, err := os.ReadFile(applicationYaml) if err != nil { return Application{}, errors.New("Failed to open application yaml file " + applicationYaml + ". " + err.Error()) @@ -25,7 +29,10 @@ func GetArgoApplication(applicationYaml string) (Application, error) { return *argoApplication, nil } -func GetArgoApplicationFromGit(gitRepository string, applicationYaml string) (Application, error) { +func GetArgoApplicationFromGit(gitRepository string, applicationYaml string) ( + Application, + error, +) { baseUrl := gitRepository + strings.TrimPrefix(applicationYaml, "../") response, err := http.Get(baseUrl) if err != nil { @@ -49,3 +56,21 @@ func GetArgoApplicationFromGit(gitRepository string, applicationYaml string) (Ap return *argoApplication, nil } + +func GatherArgoAppPaths(app Application) ( + pathCollection []string, +) { + if app.Spec.Sources != nil { + for _, source := range app.Spec.Sources { + if source.Path != "" { + pathCollection = append(pathCollection, source.Path) + } + } + } + + if app.Spec.Source != nil && app.Spec.Source.Path != "" { + pathCollection = append(pathCollection, app.Spec.Source.Path) + } + + return pathCollection +} diff --git a/pkg/client.go b/pkg/client.go new file mode 100644 index 00000000..e0dfe91e --- /dev/null +++ b/pkg/client.go @@ -0,0 +1,40 @@ +package e2eutils + +import ( + "k8s.io/client-go/discovery" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/pkg/envconf" +) + +func GetClient() klient.Client { + cfg := envconf.Config{} + return cfg.Client() +} + +func GetRestConfig() *rest.Config { + client := GetClient() + return client.RESTConfig() +} + +func GetClientSet() (*kubernetes.Clientset, error) { + kubeConfig := GetRestConfig() + clientSet, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + return nil, err + } + + return clientSet, nil +} + +func GetDynClient() (*dynamic.DynamicClient, error) { + kubeConfig := GetRestConfig() + return dynamic.NewForConfig(kubeConfig) +} + +func GetDiscoveryClient() (*discovery.DiscoveryClient, error) { + kubeConfig := GetRestConfig() + return discovery.NewDiscoveryClientForConfig(kubeConfig) +} diff --git a/test/pkg/git/git.go b/pkg/git.go similarity index 96% rename from test/pkg/git/git.go rename to pkg/git.go index f82ce500..4a785713 100644 --- a/test/pkg/git/git.go +++ b/pkg/git.go @@ -1,4 +1,4 @@ -package git +package e2eutils import ( "os/exec" diff --git a/pkg/helm/helm.go b/pkg/helm/helm.go new file mode 100644 index 00000000..b5c99643 --- /dev/null +++ b/pkg/helm/helm.go @@ -0,0 +1,72 @@ +package helm + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "e2eutils/pkg/argo" + + "sigs.k8s.io/e2e-framework/third_party/helm" +) + +func NewHelmManager(kubeConfigFile string) *helm.Manager { + return helm.New(kubeConfigFile) +} + +func AddHelmRepository(helmMgr *helm.Manager, repoURL, chartName string) error { + return helmMgr.RunRepo(helm.WithArgs("add", "--force-update", chartName, repoURL)) +} + +func DeployHelmChart(helmMgr *helm.Manager, src argo.ApplicationSource, namespace string) error { + opts, err := buildHelmOptions(src, namespace) + if err != nil { + return fmt.Errorf("building helm options: %w", err) + } + + if err := helmMgr.RunUpgrade(opts...); err != nil { + return fmt.Errorf("helm upgrade/install failed: %w", err) + } + + return nil +} + +func buildHelmOptions(src argo.ApplicationSource, namespace string) ([]helm.Option, error) { + chart := "" + if strings.HasPrefix(src.RepoURL, "oci://") { + base := strings.TrimSuffix(src.RepoURL, "/") + chart = fmt.Sprintf("%s/%s", base, src.Chart) + } else { + chart = fmt.Sprintf("%s/%s", src.Chart, src.Chart) + } + + options := []helm.Option{ + helm.WithName(src.Chart), + helm.WithNamespace(namespace), + helm.WithChart(chart), + helm.WithVersion(src.TargetRevision), + helm.WithArgs("--install", "--create-namespace"), + } + + if src.Helm != nil && src.Helm.Values != "" { + valuesFilePath, err := writeValuesFile(src.Chart, src.TargetRevision, src.Helm.Values) + if err != nil { + return nil, fmt.Errorf("writing values file: %w", err) + } + + options = append(options, helm.WithArgs("-f", valuesFilePath)) + } + + return options, nil +} + +func writeValuesFile(chart, revision, valuesContent string) (string, error) { + valuesFilePath := fmt.Sprintf("values-%s-%s.yaml", chart, revision) + filePath := filepath.Join(os.TempDir(), valuesFilePath) + if err := os.WriteFile(filePath, []byte(valuesContent), 0o600); err != nil { + return "", err + } + + return filePath, nil +} diff --git a/pkg/kustomize.go b/pkg/kustomize.go new file mode 100644 index 00000000..a7987508 --- /dev/null +++ b/pkg/kustomize.go @@ -0,0 +1,55 @@ +package e2eutils + +import ( + "bytes" + "errors" + "fmt" + "io" + "strings" + + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/kustomize/api/krusty" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +func BuildKustomization(path string) ([]*unstructured.Unstructured, error) { + if strings.TrimSpace(path) == "" { + return nil, errors.New("kustomization path must not be empty") + } + + fs := filesys.MakeFsOnDisk() + kustomizer := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) + + resMap, err := kustomizer.Run(fs, path) + if err != nil { + return nil, fmt.Errorf("failed to run kustomize: %w", err) + } + + yamlData, err := resMap.AsYaml() + if err != nil { + return nil, fmt.Errorf("failed to convert resource map to YAML: %w", err) + } + + decoder := yaml.NewDecoder(bytes.NewReader(yamlData)) + + var objects []*unstructured.Unstructured + for { + var obj unstructured.Unstructured + if err := decoder.Decode(&obj.Object); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("failed to decode YAML: %w", err) + } + + // skip empty documents + if len(obj.Object) == 0 { + continue + } + + objects = append(objects, &obj) + } + + return objects, nil +} diff --git a/pkg/manifest.go b/pkg/manifest.go new file mode 100644 index 00000000..af1ac4c3 --- /dev/null +++ b/pkg/manifest.go @@ -0,0 +1,44 @@ +package e2eutils + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "sigs.k8s.io/e2e-framework/klient/decoder" + "sigs.k8s.io/e2e-framework/klient/k8s" +) + +func GetKubernetesManifests(ctx context.Context, pathCollection []string) ([]k8s.Object, error) { + if pathCollection == nil { + return nil, errors.New("kustomization pathCollection must not be empty") + } + + var objects []k8s.Object + for _, source := range pathCollection { + if source == "" { + continue + } + + o, err := prepareKubernetesManifests(ctx, source) + if err != nil { + return nil, fmt.Errorf("failed to prepare manifests from source pathCollection %q: %w", source, err) + } + objects = append(objects, o...) + } + + return objects, nil +} + +func prepareKubernetesManifests(ctx context.Context, path string) ([]k8s.Object, error) { + manifestPath := filepath.Join("..", path) + manifestFS := os.DirFS(manifestPath) + + objects, err := decoder.DecodeAllFiles(ctx, manifestFS, "*.yaml") + if err != nil { + return nil, fmt.Errorf("failed to decode YAML files from path %q: %w", manifestPath, err) + } + return objects, nil +} diff --git a/test/pkg/test/test.go b/pkg/test.go similarity index 69% rename from test/pkg/test/test.go rename to pkg/test.go index 3f359c29..d2aee34d 100644 --- a/test/pkg/test/test.go +++ b/pkg/test.go @@ -1,8 +1,13 @@ -package test +package e2eutils import ( "context" - "errors" + "e2eutils/pkg/argo" + "e2eutils/pkg/helm" + "reflect" + "strings" + "time" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -10,100 +15,97 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/rest" - "log/slog" - "reflect" "sigs.k8s.io/e2e-framework/klient" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/klient/wait" "sigs.k8s.io/e2e-framework/klient/wait/conditions" - "sigs.k8s.io/e2e-framework/pkg/envconf" - "strings" - "test/test/pkg/argo" - "test/test/pkg/git" - "test/test/pkg/helm" - "test/test/pkg/manifest" - "time" ) -func PrepareTest(gitRepository string, applicationYaml string) (argo.Application, argo.Application, []k8s.Object, error) { - currGitBranch, err := git.GetCurrentGitBranch() - if err != nil { - return argo.Application{}, argo.Application{}, nil, err - } - - if currGitBranch == "main" { - current, err := argo.GetArgoApplication(applicationYaml) - if err != nil { - return argo.Application{}, argo.Application{}, nil, err - } +type ArgoTest struct { + Current AppSettings + Update AppSettings +} - objects, err := manifest.GetKubernetesManifests(current) - if err != nil { - return current, argo.Application{}, nil, err - } +type AppSettings struct { + Argo argo.Application + Objects []k8s.Object +} - return current, argo.Application{}, objects, nil +func PrepareArgoApp(ctx context.Context, gitRepository string, applicationYaml string) ( + ArgoTest, + error, +) { + workingBranch, err := GetCurrentGitBranch() + if err != nil { + return ArgoTest{}, err } - update, err := argo.GetArgoApplication(applicationYaml) + currentSettings, err := buildAppNode(ctx, applicationYaml, gitRepository, "main") if err != nil { - return argo.Application{}, argo.Application{}, nil, err + return ArgoTest{}, err } - objects, err := manifest.GetKubernetesManifests(update) - if err != nil { - return argo.Application{}, argo.Application{}, nil, err + argoTest := ArgoTest{ + Current: currentSettings, + } + + // when we are working on the main branch, we don't want to fetch the Update state + if workingBranch == "main" { + return argoTest, nil } - current, err := argo.GetArgoApplicationFromGit(gitRepository, applicationYaml) + argoTest.Update, err = buildAppNode(ctx, applicationYaml, gitRepository, workingBranch) if err != nil { - slog.Warn( - "Failed to get current application from git", - "application", applicationYaml, - "branch", currGitBranch, - "error", err.Error(), - ) - return update, argo.Application{}, objects, nil + return ArgoTest{}, err } - if reflect.DeepEqual(current, update) { - return current, argo.Application{}, objects, nil + // if both nodes are equal, we don't want to test the update state + if reflect.DeepEqual(argoTest.Current, argoTest.Update) { + argoTest.Update = AppSettings{} } - return current, update, objects, nil + return argoTest, nil } -func deployHelmChart(applicationSource argo.ApplicationSource, namespace string, kubeConfigFile string) error { - helmMgr := helm.GetHelmManager(kubeConfigFile) - - if !strings.Contains(applicationSource.RepoURL, "oci://") { - err := helm.AddHelmRepository(helmMgr, applicationSource.RepoURL, applicationSource.Chart) - if err != nil { - return err - } +func buildAppNode(ctx context.Context, applicationYaml string, gitRepository string, branch string) ( + AppSettings, + error, +) { + var app argo.Application + var err error + + if branch == "main" { + app, err = argo.GetArgoApplicationFromGit(gitRepository, applicationYaml) + } else { + app, err = argo.GetArgoApplication(applicationYaml) } - err := helm.DeployHelmChart(helmMgr, applicationSource, namespace) if err != nil { - return err + return AppSettings{}, err } - return nil + pathCollection := argo.GatherArgoAppPaths(app) + var objects []k8s.Object + if len(pathCollection) != 0 { + objects, err = GetKubernetesManifests(ctx, pathCollection) + if err != nil { + return AppSettings{}, err + } + } + + return AppSettings{ + Argo: app, + Objects: objects, + }, nil } func DeployHelmCharts(kubeConfigFile string, argoApplication argo.Application) error { - if argoApplication.Spec.Source != nil { - if argoApplication.Spec.Source.Chart == "" { - return nil - } - + if argoApplication.Spec.Source != nil && argoApplication.Spec.Source.Chart != "" { err := deployHelmChart(*argoApplication.Spec.Source, argoApplication.Spec.Destination.Namespace, kubeConfigFile) if err != nil { - return errors.New(err.Error()) + return err } return nil @@ -124,38 +126,26 @@ func DeployHelmCharts(kubeConfigFile string, argoApplication argo.Application) e return nil } -func GetClient() (klient.Client, error) { - cfg := envconf.Config{} - return cfg.Client(), nil -} +func deployHelmChart(applicationSource argo.ApplicationSource, namespace string, kubeConfigFile string) error { + helmMgr := helm.NewHelmManager(kubeConfigFile) -func GetRestConfig() *rest.Config { - client, _ := GetClient() - return client.RESTConfig() -} + if !strings.HasPrefix(applicationSource.RepoURL, "oci://") { + err := helm.AddHelmRepository(helmMgr, applicationSource.RepoURL, applicationSource.Chart) + if err != nil { + return err + } + } -func GetClientSet() (*kubernetes.Clientset, error) { - kubeConfig := GetRestConfig() - clientSet, err := kubernetes.NewForConfig(kubeConfig) + err := helm.DeployHelmChart(helmMgr, applicationSource, namespace) if err != nil { - return nil, err + return err } - return clientSet, nil -} - -func GetDynClient() (*dynamic.DynamicClient, error) { - kubeConfig := GetRestConfig() - return dynamic.NewForConfig(kubeConfig) -} - -func GetDiscoveryClient() (*discovery.DiscoveryClient, error) { - kubeConfig := GetRestConfig() - return discovery.NewDiscoveryClientForConfig(kubeConfig) + return nil } func CheckJobsCompleted(ctx context.Context, client klient.Client, namespace string) error { - kubeConfig := client.RESTConfig() + kubeConfig := GetRestConfig() clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return err @@ -181,7 +171,7 @@ func CheckJobsCompleted(ctx context.Context, client klient.Client, namespace str } func DeploymentBecameReady(ctx context.Context, client klient.Client, namespace string) error { - kubeConfig := client.RESTConfig() + kubeConfig := GetRestConfig() clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return err @@ -212,13 +202,13 @@ func DeploymentBecameReady(ctx context.Context, client klient.Client, namespace } func DaemonSetBecameReady(ctx context.Context, client klient.Client, namespace string) error { - kubeConfig := client.RESTConfig() + kubeConfig := GetRestConfig() clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return err } - daemonSetList, err := clientSet.AppsV1().DaemonSets(namespace).List(context.TODO(), metav1.ListOptions{}) + daemonSetList, err := clientSet.AppsV1().DaemonSets(namespace).List(ctx, metav1.ListOptions{}) if err != nil { return err } @@ -243,7 +233,7 @@ func DaemonSetBecameReady(ctx context.Context, client klient.Client, namespace s } func PersistentVolumeClaimIsBound(ctx context.Context, client klient.Client, namespace string) error { - kubeConfig := client.RESTConfig() + kubeConfig := GetRestConfig() clientSet, err := kubernetes.NewForConfig(kubeConfig) if err != nil { return err @@ -274,7 +264,7 @@ func PersistentVolumeClaimIsBound(ctx context.Context, client klient.Client, nam } func SnapshotIsReadyToUse(ctx context.Context, client klient.Client, namespace string) error { - kubeConfig := client.RESTConfig() + kubeConfig := GetRestConfig() dynClient, err := dynamic.NewForConfig(kubeConfig) if err != nil { return err diff --git a/test/argo_cd_test.go b/test/argo_cd_test.go index b7b7b08c..e60fa05d 100644 --- a/test/argo_cd_test.go +++ b/test/argo_cd_test.go @@ -1,72 +1,70 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestArgoCd(t *testing.T) { - current, update, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/argo-cd.yaml") - + argoTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/argo-cd.yaml") if err != nil { - t.Fatalf("Failed to prepare test #%v", err) + t.Fatalf("Failed to prepare test: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Argo CD Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), current) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), argoTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, current.Spec.Destination.Namespace) + err := e2eutils.DeploymentBecameReady(ctx, client, argoTest.Current.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx }). Assess("Jobs run successfully", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.CheckJobsCompleted(ctx, client, current.Spec.Destination.Namespace) + err := e2eutils.CheckJobsCompleted(ctx, client, argoTest.Current.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx }). Feature() + upgrade := features. New("Upgrading Argo CD Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if update.Spec.Sources == nil { + if argoTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err := test.DeployHelmCharts(cfg.KubeconfigFile(), update) + err := e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), argoTest.Update.Argo) assert.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, update.Spec.Destination.Namespace) + err := e2eutils.DeploymentBecameReady(ctx, client, argoTest.Update.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx }). Assess("Jobs run successfully", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.CheckJobsCompleted(ctx, client, update.Spec.Destination.Namespace) + err := e2eutils.CheckJobsCompleted(ctx, client, argoTest.Update.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx diff --git a/test/argo_rollouts_test.go b/test/argo_rollouts_test.go index 042bb21b..48e1178b 100644 --- a/test/argo_rollouts_test.go +++ b/test/argo_rollouts_test.go @@ -1,38 +1,36 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestArgoRollouts(t *testing.T) { - current, update, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/argo-rollouts.yaml") + rolloutTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/argo-rollouts.yaml") if err != nil { - t.Fatalf("Failed to prepare test #%v", err) + t.Fatalf("Failed to prepare test: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Argo Rollouts Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), current) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), rolloutTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, current.Spec.Destination.Namespace) + err := e2eutils.DeploymentBecameReady(ctx, client, rolloutTest.Current.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx @@ -42,18 +40,18 @@ func TestArgoRollouts(t *testing.T) { upgrade := features. New("Upgrading Argo Rollouts Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if update.Spec.Source == nil { + if rolloutTest.Update.Argo.Spec.Source == nil { t.SkipNow() } - err := test.DeployHelmCharts(cfg.KubeconfigFile(), update) + err := e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), rolloutTest.Update.Argo) assert.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, update.Spec.Destination.Namespace) + err := e2eutils.DeploymentBecameReady(ctx, client, rolloutTest.Update.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx diff --git a/test/cert_manager_test.go b/test/cert_manager_test.go index acaf1940..b590bf4c 100644 --- a/test/cert_manager_test.go +++ b/test/cert_manager_test.go @@ -1,56 +1,55 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestCertManager(t *testing.T) { - current, update, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/cert-manager.yaml") + certTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/cert-manager.yaml") if err != nil { - t.Fatalf("Failed to prepare test #%v", err) + t.Fatalf("Failed to prepare test: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Cert Manager Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), current) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), certTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployment became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, current.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, certTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Feature() + upgrade := features. New("Upgrading Cert Manager Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if update.Spec.Sources == nil { + if certTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DeployHelmCharts(cfg.KubeconfigFile(), update) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), certTest.Update.Argo) require.NoError(t, err) return ctx }). Assess("Testing Cert Manager upgrade became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, update.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, certTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx diff --git a/test/cloudflare_tunnel_test.go b/test/cloudflare_tunnel_test.go index 28199805..e8f419e7 100644 --- a/test/cloudflare_tunnel_test.go +++ b/test/cloudflare_tunnel_test.go @@ -1,8 +1,12 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" "fmt" + "strings" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/apps/v1" @@ -10,10 +14,6 @@ import ( "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "strings" - "test/test/pkg/api" - "test/test/pkg/test" - "testing" ) func TestCloudflareTunnel(t *testing.T) { @@ -24,27 +24,24 @@ metadata: name: %s ` - current, _, additionalManifests, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/cloudflare-tunnel.yaml") + cloudflareTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/cloudflare-tunnel.yaml") if err != nil { - t.Fatalf("Failed to prepare test #%v", err) + t.Fatalf("Failed to prepare test: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() - clientSet, err := test.GetClientSet() + clientSet, err := e2eutils.GetClientSet() if err != nil { - t.Fatalf("Failed to get kubernetes clientSet #%v", err) + t.Fatalf("Failed to get kubernetes clientSet: %v", err) } var objectList []k8s.Object install := features. New("Preparing Cloudflare Tunnel Test"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - cloudflareTunnelDeployment := additionalManifests[0].(*v1.Deployment) - cloudflareTunnelDeployment.ObjectMeta.Namespace = current.Spec.Destination.Namespace + cloudflareTunnelDeployment := cloudflareTest.Current.Objects[0].(*v1.Deployment) + cloudflareTunnelDeployment.ObjectMeta.Namespace = cloudflareTest.Current.Argo.Spec.Destination.Namespace // delete initContainers, we do not have the tunnel token in ci cloudflareTunnelDeployment.Spec.Template.Spec.InitContainers = nil @@ -57,10 +54,10 @@ metadata: objectList, err = decoder.DecodeAll(ctx, strings.NewReader(initYaml)) if err != nil { - t.Fatalf("Failed to decode namespace #%v", err) + t.Fatalf("Failed to decode namespace: %v", err) } - objectList = append(objectList, additionalManifests[0]) + objectList = append(objectList, cloudflareTest.Current.Objects[0]) return ctx }). @@ -70,14 +67,14 @@ metadata: t.Fatalf("No objects to deploy %v", objectList) } - err = api.ApplyAll(*clientSet, objectList) + err = e2eutils.ApplyAll(*clientSet, objectList) require.NoError(t, err) return ctx }). Assess("Deployment became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, current.Spec.Destination.Namespace) + err := e2eutils.DeploymentBecameReady(ctx, client, cloudflareTest.Current.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx diff --git a/test/http_echo_server_test.go b/test/http_echo_server_test.go index c7d2a844..0ac17f19 100644 --- a/test/http_echo_server_test.go +++ b/test/http_echo_server_test.go @@ -1,18 +1,18 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "os" + "strings" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "os" "sigs.k8s.io/e2e-framework/klient/decoder" "sigs.k8s.io/e2e-framework/klient/k8s" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "strings" - "test/test/pkg/api" - "test/test/pkg/test" - "testing" ) func TestHttpEchoServer(t *testing.T) { @@ -22,15 +22,9 @@ kind: Namespace metadata: name: http-echo-server ` - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } - - clientSet, err := test.GetClientSet() - if err != nil { - t.Fatalf("Failed to get kubernetes clientSet #%v", err) - } + client := e2eutils.GetClient() + clientSet, err := e2eutils.GetClientSet() + require.NoError(t, err) var objectList []k8s.Object install := features. @@ -55,14 +49,14 @@ metadata: t.Fatalf("No objects to deploy %v", objectList) } - err = api.ApplyAll(*clientSet, objectList) + err = e2eutils.ApplyAll(*clientSet, objectList) require.NoError(t, err) return ctx }). Assess("Deployment became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, "http-echo-server") + err := e2eutils.DeploymentBecameReady(ctx, client, "http-echo-server") assert.NoError(t, err) return ctx diff --git a/test/istio_test.go b/test/istio_test.go index 68b47ea3..759df061 100644 --- a/test/istio_test.go +++ b/test/istio_test.go @@ -1,51 +1,49 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "log" + "testing" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" - "log" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestIstio(t *testing.T) { - istioCurrent, istioUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/istio.yaml") + istioTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/istio.yaml") if err != nil { - t.Fatalf("Failed to prepare test #%v", err) + t.Fatalf("Failed to prepare test: %v", err) } - gatewayCurrent, gatewayUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/istio-gateway.yaml") + gatewayTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/istio-gateway.yaml") if err != nil { t.Fatalf("Failed to prepare test #%v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Istio Helm Charts Collection"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), istioCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), istioTest.Current.Argo) require.NoError(t, err) - err = test.DeployHelmCharts(cfg.KubeconfigFile(), gatewayCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), gatewayTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, istioCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, istioTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) - err = test.DeploymentBecameReady(ctx, client, gatewayCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, gatewayTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx @@ -95,24 +93,24 @@ func TestIstio(t *testing.T) { upgrade := features. New("Upgrading Istio Helm Charts Collection"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if istioUpdate.Spec.Sources == nil && gatewayUpdate.Spec.Sources == nil { + if istioTest.Update.Argo.Spec.Sources == nil && gatewayTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DeployHelmCharts(cfg.KubeconfigFile(), istioUpdate) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), istioTest.Update.Argo) require.NoError(t, err) - err = test.DeployHelmCharts(cfg.KubeconfigFile(), gatewayUpdate) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), gatewayTest.Update.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, istioUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, istioTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) - err = test.DeploymentBecameReady(ctx, client, gatewayUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, gatewayTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx diff --git a/test/k3s_system_upgrade_controller_test.go b/test/k3s_system_upgrade_controller_test.go index d2779be8..9e905640 100644 --- a/test/k3s_system_upgrade_controller_test.go +++ b/test/k3s_system_upgrade_controller_test.go @@ -1,39 +1,36 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "time" + "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "sigs.k8s.io/yaml" - "test/test/pkg/api" - "test/test/pkg/manifest" - "test/test/pkg/test" - "testing" - "time" ) func TestK3sSystemUpgradeController(t *testing.T) { - current, update, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/k3s-system-upgrade-controller.yaml") + upgradeTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/k3s-system-upgrade-controller.yaml") require.NoError(t, err) - clientSet, err := test.GetClientSet() + clientSet, err := e2eutils.GetClientSet() require.NoError(t, err) - client, err := test.GetClient() - require.NoError(t, err) + client := e2eutils.GetClient() - var kustomization []string + var kustomization []*unstructured.Unstructured var namespace string install := features. New("Kustomization"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if update.Spec.Sources != nil { - kustomization, err = manifest.BuildKustomization(update.Spec.Sources[0].Path) + if upgradeTest.Update.Argo.Spec.Sources != nil { + kustomization, err = e2eutils.BuildKustomization("../" + upgradeTest.Update.Argo.Spec.Sources[0].Path) } else { - kustomization, err = manifest.BuildKustomization(current.Spec.Sources[0].Path) + kustomization, err = e2eutils.BuildKustomization("../" + upgradeTest.Current.Argo.Spec.Sources[0].Path) } require.NoError(t, err) @@ -41,33 +38,25 @@ func TestK3sSystemUpgradeController(t *testing.T) { }). Assess("Deployment", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - for _, resource := range kustomization { - var object unstructured.Unstructured - err = yaml.Unmarshal([]byte(resource), &object.Object) - require.NoError(t, err) - + // deploy namespace + for _, object := range kustomization { if object.GetKind() != "Namespace" { continue } - namespace = object.GetName() - err = api.Apply(*clientSet, &object) + err = e2eutils.Apply(*clientSet, object) require.NoError(t, err) // give k8s api some time to create a resource time.Sleep(100 * time.Millisecond) } - for _, resource := range kustomization { - var object unstructured.Unstructured - err = yaml.Unmarshal([]byte(resource), &object.Object) - require.NoError(t, err) - + for _, object := range kustomization { if object.GetKind() == "Namespace" { continue } - err = api.Apply(*clientSet, &object) + err = e2eutils.Apply(*clientSet, object) require.NoError(t, err) // give k8s api some time to create a resource @@ -78,7 +67,7 @@ func TestK3sSystemUpgradeController(t *testing.T) { }). Assess("Deployment became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, namespace) require.NoError(t, err) return ctx diff --git a/test/kargo_test.go b/test/kargo_test.go deleted file mode 100644 index 501fe43f..00000000 --- a/test/kargo_test.go +++ /dev/null @@ -1,79 +0,0 @@ -package test - -import ( - "context" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "sigs.k8s.io/e2e-framework/pkg/envconf" - "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" -) - -func TestKargo(t *testing.T) { - kargoCurrent, kargoUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/kargo.yaml") - if err != nil { - t.Fatalf("Failed to prepare kargo test #%v", err) - } - - certCurrent, _, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/cert-manager.yaml") - if err != nil { - t.Fatalf("Failed to prepare cert-manager #%v", err) - } - - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } - - install := features. - New("Deploying Kargo.io Helm Chart"). - Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - for index, _ := range kargoCurrent.Spec.Sources { - kargoCurrent.Spec.Sources[index].RepoURL = "oci://" + kargoCurrent.Spec.Sources[index].RepoURL + "/" + kargoCurrent.Spec.Sources[index].Chart - } - - // kargo depends on cert-manager crds - err = test.DeployHelmCharts(cfg.KubeconfigFile(), certCurrent) - require.NoError(t, err) - - err = test.DeployHelmCharts(cfg.KubeconfigFile(), kargoCurrent) - require.NoError(t, err) - - return ctx - }). - Assess("Deployments became ready", - func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, kargoCurrent.Spec.Destination.Namespace) - assert.NoError(t, err) - - return ctx - }). - Feature() - upgrade := features. - New("Upgrading Kargo.io Helm Chart"). - Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if kargoUpdate.Spec.Sources == nil { - t.SkipNow() - } - - for index, _ := range kargoUpdate.Spec.Sources { - kargoUpdate.Spec.Sources[index].RepoURL = "oci://" + kargoUpdate.Spec.Sources[index].RepoURL + "/" + kargoUpdate.Spec.Sources[index].Chart - } - - err := test.DeployHelmCharts(cfg.KubeconfigFile(), kargoUpdate) - assert.NoError(t, err) - - return ctx - }). - Assess("Deployments became ready", - func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DeploymentBecameReady(ctx, client, kargoUpdate.Spec.Destination.Namespace) - assert.NoError(t, err) - - return ctx - }). - Feature() - - ciTestEnv.Test(t, install, upgrade) -} diff --git a/test/kured_test.go b/test/kured_test.go index c68eb3b5..d2f67406 100644 --- a/test/kured_test.go +++ b/test/kured_test.go @@ -1,33 +1,33 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestKured(t *testing.T) { - current, update, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/kured.yaml") + kuredTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/kured.yaml") require.NoError(t, err) - client, err := test.GetClient() - require.NoError(t, err) + client := e2eutils.GetClient() install := features. New("Deploying Kured Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), current) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), kuredTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("DaemonSet became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DaemonSetBecameReady(ctx, client, current.Spec.Destination.Namespace) + err := e2eutils.DaemonSetBecameReady(ctx, client, kuredTest.Current.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx @@ -37,18 +37,18 @@ func TestKured(t *testing.T) { upgrade := features. New("Upgrading Kured Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if update.Spec.Source == nil { + if kuredTest.Update.Argo.Spec.Source == nil { t.SkipNow() } - err := test.DeployHelmCharts(cfg.KubeconfigFile(), update) + err := e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), kuredTest.Update.Argo) assert.NoError(t, err) return ctx }). Assess("DaemonSet became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err := test.DaemonSetBecameReady(ctx, client, update.Spec.Destination.Namespace) + err := e2eutils.DaemonSetBecameReady(ctx, client, kuredTest.Update.Argo.Spec.Destination.Namespace) assert.NoError(t, err) return ctx diff --git a/test/main_test.go b/test/main_test.go index 6d93da64..25777265 100644 --- a/test/main_test.go +++ b/test/main_test.go @@ -1,13 +1,14 @@ -package test +package e2eutils import ( "fmt" "os" + "testing" + "sigs.k8s.io/e2e-framework/pkg/env" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/envfuncs" "sigs.k8s.io/e2e-framework/support/kind" - "testing" ) var ( diff --git a/test/pkg/helm/helm.go b/test/pkg/helm/helm.go deleted file mode 100644 index f9618954..00000000 --- a/test/pkg/helm/helm.go +++ /dev/null @@ -1,113 +0,0 @@ -package helm - -import ( - "fmt" - "os" - "sigs.k8s.io/e2e-framework/third_party/helm" - "strings" - "test/test/pkg/argo" -) - -type HelmOptions struct { - Name string `default:""` - Namespace string `default:""` - Chart string `default:""` - Version string `default:""` - Values string `default:""` - OciRepository string `default:""` -} - -func GetHelmManager(kubeConfigFile string) *helm.Manager { - return helm.New(kubeConfigFile) -} - -func AddHelmRepository(helmMgr *helm.Manager, helmRepoUrl string, helmChartName string) error { - err := helmMgr.RunRepo(helm.WithArgs( - "add --force-update", - helmChartName, - helmRepoUrl, - )) - if err != nil { - return err - } - - return nil -} - -func helmifyApp(app argo.ApplicationSource, namespace string) (HelmOptions, error) { - fullChartName := getFullChartName(app.Chart, app.Chart) - helmOciRepository := "" - - if strings.Contains(app.RepoURL, "oci://") { - fullChartName = "" - helmOciRepository = app.RepoURL - } - - helmValues := make(map[int]string, 2) - if app.Helm != nil { - err := helmValuesToFile(app) - if err != nil { - return HelmOptions{}, err - } - - helmValues[0] = "-f" - helmValues[1] = "/tmp/helm-values.txt" - } - - helmOptions := HelmOptions{ - Name: app.Chart, - Chart: fullChartName, - Namespace: namespace, - Version: app.TargetRevision, - Values: helmValues[0] + " " + helmValues[1], - OciRepository: helmOciRepository, - } - - return helmOptions, nil -} - -func getFullChartName(helmRepoName string, helmChart string) string { - return fmt.Sprintf("%s/%s", helmRepoName, helmChart) -} - -func helmValuesToFile(applicationSource argo.ApplicationSource) error { - helmValues, err := os.Create("/tmp/helm-values.txt") - if err != nil { - return err - } - - _, err = helmValues.WriteString(applicationSource.Helm.Values) - if err != nil { - return err - } - - err = helmValues.Close() - if err != nil { - return err - } - - return nil -} - -func DeployHelmChart(helmMgr *helm.Manager, applicationSource argo.ApplicationSource, namespace string) error { - helmOptions, err := helmifyApp(applicationSource, namespace) - if err != nil { - return err - } - - err = helmMgr.RunUpgrade( - helm.WithArgs("--install"), - helm.WithName(helmOptions.Name), - helm.WithNamespace(helmOptions.Namespace), - helm.WithChart(helmOptions.Chart), - helm.WithVersion(helmOptions.Version), - helm.WithArgs("--create-namespace"), - helm.WithArgs(helmOptions.Values), - helm.WithArgs(helmOptions.OciRepository), - ) - if err != nil { - return err - } - - return nil -} diff --git a/test/pkg/manifest/manifest.go b/test/pkg/manifest/manifest.go deleted file mode 100644 index b3cf68e1..00000000 --- a/test/pkg/manifest/manifest.go +++ /dev/null @@ -1,70 +0,0 @@ -package manifest - -import ( - "context" - "os" - "sigs.k8s.io/e2e-framework/klient/decoder" - "sigs.k8s.io/e2e-framework/klient/k8s" - "sigs.k8s.io/kustomize/api/krusty" - "sigs.k8s.io/kustomize/kyaml/filesys" - "strings" - "test/test/pkg/argo" -) - -func GetKubernetesManifests(argoApplication argo.Application) ([]k8s.Object, error) { - var objects []k8s.Object - var err error - - if argoApplication.Spec.Source != nil { - if argoApplication.Spec.Source.Path == "" { - return nil, nil - } - - objects, err = prepareKubernetesManifests(*argoApplication.Spec.Source) - if err != nil { - return nil, err - } - } - - var source argo.ApplicationSource - for _, source = range argoApplication.Spec.Sources { - if source.Path == "" { - continue - } - - objects, err = prepareKubernetesManifests(source) - if err != nil { - return nil, err - } - } - - return objects, nil -} - -func prepareKubernetesManifests(applicationSource argo.ApplicationSource) ([]k8s.Object, error) { - realPath := os.DirFS("../" + applicationSource.Path) - - objects, err := decoder.DecodeAllFiles(context.TODO(), realPath, "*.yaml") - if err != nil { - return nil, err - } - return objects, nil -} - -func BuildKustomization(path string) ([]string, error) { - fSys := filesys.MakeFsOnDisk() - kustomizationDir := "../" + path - k := krusty.MakeKustomizer(krusty.MakeDefaultOptions()) - - objects, err := k.Run(fSys, kustomizationDir) - if err != nil { - return nil, err - } - - yaml, err := objects.AsYaml() - if err != nil { - return nil, err - } - - return strings.Split(string(yaml), "---"), nil -} diff --git a/test/prometheus_test.go b/test/prometheus_test.go index dabbfbdf..991ad4b8 100644 --- a/test/prometheus_test.go +++ b/test/prometheus_test.go @@ -1,22 +1,23 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "strings" + "testing" + "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "strings" - "test/test/pkg/test" - "testing" ) func TestPrometheus(t *testing.T) { - promCurrent, promUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/prometheus.yaml") + prometheusTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/prometheus.yaml") if err != nil { t.Fatalf("Failed to prepare prometheus test #%v", err) } - for i, source := range promCurrent.Spec.Sources { + for i, source := range prometheusTest.Current.Argo.Spec.Sources { if source.Chart == "" { continue } @@ -50,10 +51,10 @@ func TestPrometheus(t *testing.T) { -1, ) - promCurrent.Spec.Sources[i].Helm.Values = source.Helm.Values + prometheusTest.Current.Argo.Spec.Sources[i].Helm.Values = source.Helm.Values } - for i, source := range promUpdate.Spec.Sources { + for i, source := range prometheusTest.Update.Argo.Spec.Sources { if source.Chart == "" { continue } @@ -87,32 +88,29 @@ func TestPrometheus(t *testing.T) { -1, ) - promUpdate.Spec.Sources[i].Helm.Values = source.Helm.Values + prometheusTest.Update.Argo.Spec.Sources[i].Helm.Values = source.Helm.Values } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Prometheus Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), promCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), prometheusTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, promCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, prometheusTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Daemonsets became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DaemonSetBecameReady(ctx, client, promCurrent.Spec.Destination.Namespace) + err = e2eutils.DaemonSetBecameReady(ctx, client, prometheusTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx @@ -122,25 +120,25 @@ func TestPrometheus(t *testing.T) { upgrade := features. New("Upgrading Prometheus Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if promUpdate.Spec.Sources == nil { + if prometheusTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DeployHelmCharts(cfg.KubeconfigFile(), promUpdate) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), prometheusTest.Update.Argo) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, promUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, prometheusTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Daemonsets became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DaemonSetBecameReady(ctx, client, promUpdate.Spec.Destination.Namespace) + err = e2eutils.DaemonSetBecameReady(ctx, client, prometheusTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx diff --git a/test/sealed_secrets_test.go b/test/sealed_secrets_test.go index cb7432b5..1258a9c7 100644 --- a/test/sealed_secrets_test.go +++ b/test/sealed_secrets_test.go @@ -1,36 +1,34 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "testing" + "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "test/test/pkg/test" - "testing" ) func TestSealedSecrets(t *testing.T) { - sealedCurrent, sealedUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/sealed-secrets.yaml") + sealedTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/sealed-secrets.yaml") if err != nil { - t.Fatalf("Failed to prepare sealed secret test #%v", err) + t.Fatalf("Failed to prepare sealed secret test: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() install := features. New("Deploying Sealed Secrets Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), sealedCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), sealedTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Deployment became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, sealedCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, sealedTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx @@ -39,18 +37,18 @@ func TestSealedSecrets(t *testing.T) { upgrade := features. New("Upgrading Sealed Secrets Helm Chart"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if sealedUpdate.Spec.Source == nil { + if sealedTest.Update.Argo.Spec.Source == nil { t.SkipNow() } - err = test.DeployHelmCharts(cfg.KubeconfigFile(), sealedUpdate) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), sealedTest.Update.Argo) require.NoError(t, err) return ctx }). Assess("Testing Sealed Secrets upgrade became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, sealedUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, sealedTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx diff --git a/test/storage_test.go b/test/storage_test.go index 624dda27..71be2c4f 100644 --- a/test/storage_test.go +++ b/test/storage_test.go @@ -1,15 +1,15 @@ -package test +package e2eutils import ( "context" + "e2eutils/pkg" + "strings" + "testing" + "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/klient/decoder" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" - "strings" - "test/test/pkg/api" - "test/test/pkg/test" - "testing" ) func TestStorage(t *testing.T) { @@ -28,22 +28,19 @@ spec: storageClassName: longhorn ` - scCurrent, scUpdate, _, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/snapshot-controller.yaml") + snapshotTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/snapshot-controller.yaml") if err != nil { - t.Fatalf("Failed to prepare shanpshot controller test #%v", err) + t.Fatalf("Failed to prepare shanpshot controller test: %v", err) } - longhornCurrent, longhornUpdate, manifest, err := test.PrepareTest(gitRepository, "../kubernetes-services/templates/longhorn.yaml") + longhornTest, err := e2eutils.PrepareArgoApp(t.Context(), gitRepository, "../kubernetes-services/templates/longhorn.yaml") if err != nil { - t.Fatalf("Failed to prepare longhorn csi #%v", err) + t.Fatalf("Failed to prepare longhorn csi: %v", err) } - client, err := test.GetClient() - if err != nil { - t.Fatalf("Failed to get kubernetes client #%v", err) - } + client := e2eutils.GetClient() - clientSet, err := test.GetClientSet() + clientSet, err := e2eutils.GetClientSet() if err != nil { t.Fatalf("Failed to get kubernetes clientSet #%v", err) } @@ -53,14 +50,14 @@ spec: Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { // in ci we run a single node instance of kind - longhornCurrent.Spec.Sources[0].Helm.Values = strings.Replace( - longhornCurrent.Spec.Sources[0].Helm.Values, + longhornTest.Current.Argo.Spec.Sources[0].Helm.Values = strings.Replace( + longhornTest.Current.Argo.Spec.Sources[0].Helm.Values, "defaultClassReplicaCount: 4", "defaultClassReplicaCount: 1", -1, ) - longhornCurrent.Spec.Sources[0].Helm.Values = ` + longhornTest.Current.Argo.Spec.Sources[0].Helm.Values = ` csi: attacherReplicaCount: 1 provisionerReplicaCount: 1 @@ -69,8 +66,8 @@ csi: ` // we also do not have prometheus - longhornCurrent.Spec.Sources[0].Helm.Values = strings.Replace( - longhornCurrent.Spec.Sources[0].Helm.Values, + longhornTest.Current.Argo.Spec.Sources[0].Helm.Values = strings.Replace( + longhornTest.Current.Argo.Spec.Sources[0].Helm.Values, "serviceMonitor:\n enabled: true", "serviceMonitor:\n enabled: false", -1, @@ -80,38 +77,38 @@ csi: }). Assess("Deploying CSI Helm Charts", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), scCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), snapshotTest.Current.Argo) require.NoError(t, err) - err = test.DeployHelmCharts(cfg.KubeconfigFile(), longhornCurrent) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), longhornTest.Current.Argo) require.NoError(t, err) return ctx }). Assess("Longhorn DaemonSet became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DaemonSetBecameReady(ctx, client, longhornCurrent.Spec.Destination.Namespace) + err = e2eutils.DaemonSetBecameReady(ctx, client, longhornTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, longhornCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, longhornTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Snapshot Controller Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = test.DeploymentBecameReady(ctx, client, scCurrent.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, snapshotTest.Current.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Deploy Snapshot Class", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - err = api.ApplyAll(*clientSet, manifest) + err = e2eutils.ApplyAll(*clientSet, longhornTest.Current.Objects) require.NoError(t, err) return ctx @@ -119,10 +116,10 @@ csi: Assess("Deploy PVC", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { pvcObject, err := decoder.DecodeAll(ctx, strings.NewReader(pvc)) - err = api.ApplyAll(*clientSet, pvcObject) + err = e2eutils.ApplyAll(*clientSet, pvcObject) require.NoError(t, err) - err = test.PersistentVolumeClaimIsBound(ctx, client, "default") + err = e2eutils.PersistentVolumeClaimIsBound(ctx, client, "default") require.NoError(t, err) return ctx @@ -131,25 +128,25 @@ csi: upgrade := features. New("Upgrading CSI Helm Charts"). Setup(func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if longhornUpdate.Spec.Sources == nil && scUpdate.Spec.Sources == nil { + if longhornTest.Update.Argo.Spec.Sources == nil && snapshotTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - if scUpdate.Spec.Sources != nil { - err = test.DeployHelmCharts(cfg.KubeconfigFile(), scUpdate) + if snapshotTest.Update.Argo.Spec.Sources != nil { + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), snapshotTest.Update.Argo) require.NoError(t, err) } - if longhornUpdate.Spec.Sources != nil { + if longhornTest.Update.Argo.Spec.Sources != nil { // in ci we run a single node instance of kind - longhornUpdate.Spec.Sources[0].Helm.Values = strings.Replace( - longhornUpdate.Spec.Sources[0].Helm.Values, + longhornTest.Update.Argo.Spec.Sources[0].Helm.Values = strings.Replace( + longhornTest.Update.Argo.Spec.Sources[0].Helm.Values, "defaultClassReplicaCount: 4", "defaultClassReplicaCount: 1", -1, ) - longhornUpdate.Spec.Sources[0].Helm.Values = ` + longhornTest.Update.Argo.Spec.Sources[0].Helm.Values = ` csi: attacherReplicaCount: 1 provisionerReplicaCount: 1 @@ -158,47 +155,47 @@ csi: ` // we also do not have prometheus - longhornUpdate.Spec.Sources[0].Helm.Values = strings.Replace( - longhornUpdate.Spec.Sources[0].Helm.Values, + longhornTest.Update.Argo.Spec.Sources[0].Helm.Values = strings.Replace( + longhornTest.Update.Argo.Spec.Sources[0].Helm.Values, "serviceMonitor:\n enabled: true", "serviceMonitor:\n enabled: false", -1, ) - err = test.DeployHelmCharts(cfg.KubeconfigFile(), longhornUpdate) + err = e2eutils.DeployHelmCharts(cfg.KubeconfigFile(), longhornTest.Update.Argo) require.NoError(t, err) } return ctx }). Assess("Longhorn DaemonSet became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if longhornUpdate.Spec.Sources == nil { + if longhornTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DaemonSetBecameReady(ctx, client, longhornUpdate.Spec.Destination.Namespace) + err = e2eutils.DaemonSetBecameReady(ctx, client, longhornTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Longhorn Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if longhornUpdate.Spec.Sources == nil { + if longhornTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DeploymentBecameReady(ctx, client, longhornUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, longhornTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx }). Assess("Snapshot Controller Deployments became ready", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - if scUpdate.Spec.Sources == nil { + if snapshotTest.Update.Argo.Spec.Sources == nil { t.SkipNow() } - err = test.DeploymentBecameReady(ctx, client, scUpdate.Spec.Destination.Namespace) + err = e2eutils.DeploymentBecameReady(ctx, client, snapshotTest.Update.Argo.Spec.Destination.Namespace) require.NoError(t, err) return ctx