Merge pull request #30007 from neisw/trt-2163-incluster-disruption-images

TRT-2163: incluster disruption images

Author: openshift-merge-bot[bot]

pkg/monitortests/kubeapiserver/disruptioninclusterapiserver/monitortest.go

@@ -1,20 +1,19 @@
 package disruptioninclusterapiserver
 
 import (
-	"bytes"
 	"context"
 	_ "embed"
 	"fmt"
 	"net/url"
-	"os/exec"
-	"strings"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/sirupsen/logrus"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 
 	"github.com/openshift/origin/pkg/monitortestlibrary/disruptionlibrary"
+	"github.com/openshift/origin/pkg/test/extensions"
+	"github.com/openshift/origin/test/extended/util/payload"
 	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/selection"
 
@@ -304,34 +303,27 @@ func (i *InvariantInClusterDisruption) PrepareCollection(ctx context.Context, ad
 func (i *InvariantInClusterDisruption) StartCollection(ctx context.Context, adminRESTConfig *rest.Config, _ monitorapi.RecorderWriter) error {
 	var err error
 	log := logrus.WithField("monitorTest", "apiserver-incluster-availability").WithField("namespace", i.namespaceName).WithField("func", "StartCollection")
-	log.Infof("payload image pull spec is %v", i.payloadImagePullSpec)
 	if len(i.payloadImagePullSpec) == 0 {
-		configClient, err := configclient.NewForConfig(adminRESTConfig)
+		i.payloadImagePullSpec, err = extensions.DetermineReleasePayloadImage()
 		if err != nil {
 			return err
 		}
-		clusterVersion, err := configClient.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
-		if apierrors.IsNotFound(err) {
-			i.notSupportedReason = "clusterversion/version not found and no image pull spec specified."
+
+		if len(i.payloadImagePullSpec) == 0 {
+			log.Info("unable to determine payloadImagePullSpec")
+			i.notSupportedReason = "no image pull spec specified."
 			return nil
 		}
-		if err != nil {
-			return err
-		}
-		i.payloadImagePullSpec = clusterVersion.Status.History[0].Image
 	}
 
-	// runImageExtract extracts src from specified image to dst
-	cmd := exec.Command("oc", "adm", "release", "info", i.payloadImagePullSpec, "--image-for=tests")
-	out := &bytes.Buffer{}
-	errOut := &bytes.Buffer{}
-	cmd.Stdout = out
-	cmd.Stderr = errOut
-	if err := cmd.Run(); err != nil {
-		i.notSupportedReason = fmt.Sprintf("unable to determine openshift-tests image: %v: %v", err, errOut.String())
-		return nil
+	log.Infof("payload image pull spec is %s", i.payloadImagePullSpec)
+
+	// Extract the openshift-tests image from the release payload
+	oc := exutil.NewCLIForMonitorTest("default")
+	i.openshiftTestsImagePullSpec, err = payload.ExtractImageFromReleasePayload(i.payloadImagePullSpec, "tests", oc)
+	if err != nil {
+		return fmt.Errorf("unable to determine openshift-tests image: %s: %v", i.payloadImagePullSpec, err)
 	}
-	i.openshiftTestsImagePullSpec = strings.TrimSpace(out.String())
 	log.Infof("openshift-tests image pull spec is %v", i.openshiftTestsImagePullSpec)
 
 	i.adminRESTConfig = adminRESTConfig

pkg/monitortests/network/disruptionpodnetwork/monitortest.go

@@ -91,14 +91,13 @@ func NewPodNetworkAvalibilityInvariant(info monitortestframework.MonitorTestInit
 func (pna *podNetworkAvalibility) PrepareCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
 	deploymentID := uuid.New().String()
 
-	openshiftTestsImagePullSpec, err := GetOpenshiftTestsImagePullSpec(ctx, adminRESTConfig, pna.payloadImagePullSpec, nil)
+	oc := util.NewCLIWithoutNamespace("openshift-tests")
+	openshiftTestsImagePullSpec, err := GetOpenshiftTestsImagePullSpec(ctx, adminRESTConfig, pna.payloadImagePullSpec, oc)
 	if err != nil {
 		pna.notSupportedReason = &monitortestframework.NotSupportedError{Reason: fmt.Sprintf("unable to determine openshift-tests image: %v", err)}
 		return pna.notSupportedReason
 	}
 
-	// Skip on ROSA TRT-1869
-	oc := util.NewCLIWithoutNamespace("openshift-tests")
 	isManagedServiceCluster, err := util.IsManagedServiceCluster(ctx, oc.AdminKubeClient())
 	if isManagedServiceCluster {
 		pna.notSupportedReason = &monitortestframework.NotSupportedError{Reason: fmt.Sprintf("pod network tests are unschedulable on ROSA TRT-1869")}

pkg/monitortests/network/disruptionpodnetwork/utils.go

@@ -1,20 +1,20 @@
 package disruptionpodnetwork
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
+
 	"io/ioutil"
 	"os"
-	"os/exec"
 	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
 
-	configclient "github.com/openshift/client-go/config/clientset/versioned"
+	"github.com/openshift/origin/pkg/test/extensions"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/payload"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
@@ -24,35 +24,25 @@ import (
 // IN ginkgo environment, oc needs to be created before BeforeEach and passed in
 func GetOpenshiftTestsImagePullSpec(ctx context.Context, adminRESTConfig *rest.Config, suggestedPayloadImage string, oc *exutil.CLI) (string, error) {
 	if len(suggestedPayloadImage) == 0 {
-		configClient, err := configclient.NewForConfig(adminRESTConfig)
-		if err != nil {
-			return "", err
-		}
-		clusterVersion, err := configClient.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
-		if apierrors.IsNotFound(err) {
-			return "", fmt.Errorf("clusterversion/version not found and no image pull spec specified")
-		}
+		var err error
+		suggestedPayloadImage, err = extensions.DetermineReleasePayloadImage()
 		if err != nil {
 			return "", err
 		}
-		suggestedPayloadImage = clusterVersion.Status.History[0].Image
 	}
 
 	logrus.Infof("payload image reported by CV: %v\n", suggestedPayloadImage)
 	// runImageExtract extracts src from specified image to dst
-	cmd := exec.Command("oc", "adm", "release", "info", suggestedPayloadImage, "--image-for=tests")
-	out := &bytes.Buffer{}
-	outStr := ""
-	errOut := &bytes.Buffer{}
-	cmd.Stdout = out
-	cmd.Stderr = errOut
-	if err := cmd.Run(); err != nil {
-		logrus.WithError(err).Errorf("unable to determine openshift-tests image through exec: %v", errOut.String())
+
+	// Extract the openshift-tests image from the release payload
+	openshiftTestsImagePullSpec, err := payload.ExtractImageFromReleasePayload(suggestedPayloadImage, "tests", oc)
+	if err != nil {
+		logrus.WithError(err).Errorf("unable to determine openshift-tests image through ExtractImageFromReleasePayload: %v", err)
 		// Now try the wrapper to see if it makes a difference
 		if oc == nil {
 			oc = exutil.NewCLIWithoutNamespace("openshift-tests")
 		}
-		outStr, err = oc.Run("adm", "release", "info", suggestedPayloadImage).Args("--image-for=tests").Output()
+		outStr, err := oc.Run("adm", "release", "info", suggestedPayloadImage).Args("--image-for=tests").Output()
 		if err != nil {
 			logrus.WithError(err).Errorf("unable to determine openshift-tests image through oc wrapper with default ps: %v", outStr)
 
@@ -130,11 +120,9 @@ func GetOpenshiftTestsImagePullSpec(ctx context.Context, adminRESTConfig *rest.C
 		} else {
 			logrus.Infof("successfully getting image for test with oc wrapper with default ps: %s\n", outStr)
 		}
-	} else {
-		outStr = out.String()
-	}
 
-	openshiftTestsImagePullSpec := strings.TrimSpace(outStr)
+		openshiftTestsImagePullSpec = strings.TrimSpace(outStr)
+	}
 	fmt.Printf("openshift-tests image pull spec is %v\n", openshiftTestsImagePullSpec)
 
 	return openshiftTestsImagePullSpec, nil

pkg/test/extensions/binary.go

@@ -24,8 +24,6 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
-	kapierrs "k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 	k8simage "k8s.io/kubernetes/test/utils/image"
@@ -35,7 +33,6 @@ import (
 	"github.com/openshift/origin/pkg/clioptions/clusterdiscovery"
 	"github.com/openshift/origin/pkg/clioptions/imagesetup"
 	"github.com/openshift/origin/pkg/clioptions/upgradeoptions"
-	"github.com/openshift/origin/test/extended/util"
 	exutil "github.com/openshift/origin/test/extended/util"
 	origingenerated "github.com/openshift/origin/test/extended/util/annotate/generated"
 	"github.com/openshift/origin/test/extended/util/image"
@@ -417,69 +414,22 @@ func ExtractAllTestBinaries(ctx context.Context, parallelism int) (func(), TestB
 		return nil, nil, errors.New("parallelism must be greater than zero")
 	}
 
-	releaseImage, err := determineReleasePayloadImage()
+	releaseImage, err := DetermineReleasePayloadImage()
 	if err != nil {
 		return nil, nil, errors.WithMessage(err, "couldn't determine release image")
 	}
 
-	oc := util.NewCLIWithoutNamespace("default")
-
-	// To extract binaries bearing external tests, we must inspect the release
-	// payload under tests as well as extract content from component images
-	// referenced by that payload.
-	// openshift-tests is frequently run in the context of a CI job, within a pod.
-	// CI sets $RELEASE_IMAGE_LATEST to a pullspec for the release payload under test. This
-	// pull spec resolve to:
-	// 1. A build farm ci-op-* namespace / imagestream location (anonymous access permitted).
-	// 2. A quay.io/openshift-release-dev location (for tests against promoted ART payloads -- anonymous access permitted).
-	// 3. A registry.ci.openshift.org/ocp-<arch>/release:<tag> (request registry.ci.openshift.org token).
-	// Within the pod, we don't necessarily have a pull-secret for #3 OR the component images
-	// a payload references (which are private, unless in a ci-op-* imagestream).
-	// We try the following options:
-	// 1. If set, use the REGISTRY_AUTH_FILE environment variable to an auths file with
-	//    pull secrets capable of reading appropriate payload & component image
-	//    information.
-	// 2. If it exists, use a file /run/secrets/ci.openshift.io/cluster-profile/pull-secret
-	//    (conventional location for pull-secret information for CI cluster profile).
-	// 3. Use openshift-config secret/pull-secret from the cluster-under-test, if it exists
-	//    (Microshift does not).
-	// 4. Use unauthenticated access to the payload image and component images.
-	registryAuthFilePath := os.Getenv("REGISTRY_AUTH_FILE")
-
-	// if the environment variable is not set, extract the target cluster's
-	// platform pull secret.
-	if len(registryAuthFilePath) != 0 {
-		logrus.Infof("Using REGISTRY_AUTH_FILE environment variable: %v", registryAuthFilePath)
-	} else {
-
-		// See if the cluster-profile has stored a pull-secret at the conventional location.
-		ciProfilePullSecretPath := "/run/secrets/ci.openshift.io/cluster-profile/pull-secret"
-		_, err := os.Stat(ciProfilePullSecretPath)
-		if !os.IsNotExist(err) {
-			logrus.Infof("Detected %v; using cluster profile for image access", ciProfilePullSecretPath)
-			registryAuthFilePath = ciProfilePullSecretPath
-		} else {
-			// Inspect the cluster-under-test and read its cluster pull-secret dockerconfigjson value.
-			clusterPullSecret, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-config").Get(context.Background(), "pull-secret", metav1.GetOptions{})
-			if err != nil {
-				if kapierrs.IsNotFound(err) {
-					logrus.Warningf("Cluster has no openshift-config secret/pull-secret; falling back to unauthenticated image access")
-				} else {
-					return nil, nil, fmt.Errorf("unable to read ephemeral cluster pull secret: %w", err)
-				}
-			} else {
-				tmpDir, err := os.MkdirTemp("", "external-binary")
-				clusterDockerConfig := clusterPullSecret.Data[".dockerconfigjson"]
-				registryAuthFilePath = filepath.Join(tmpDir, ".dockerconfigjson")
-				err = os.WriteFile(registryAuthFilePath, clusterDockerConfig, 0600)
-				if err != nil {
-					return nil, nil, fmt.Errorf("unable to serialize target cluster pull-secret locally: %w", err)
-				}
+	tmpDir, err := os.MkdirTemp("", "external-binary")
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to create temporary directory: %w", err)
+	}
 
-				defer os.RemoveAll(tmpDir)
-				logrus.Infof("Using target cluster pull-secrets for registry auth")
-			}
-		}
+	defer os.RemoveAll(tmpDir)
+
+	oc := exutil.NewCLIWithoutNamespace("default")
+	registryAuthFilePath, err := DetermineRegistryAuthFilePath(tmpDir, oc)
+	if err != nil {
+		return nil, nil, fmt.Errorf("failed to determine registry auth file path: %w", err)
 	}
 
 	externalBinaryProvider, err := NewExternalBinaryProvider(releaseImage, registryAuthFilePath)

pkg/test/extensions/provider.go

@@ -59,7 +59,7 @@ func NewExternalBinaryProvider(releaseImage, registryAuthfilePath string) (*Exte
 		return nil, errors.WithMessagef(err, "error creating cache path %s", binDir)
 	}
 
-	releasePayloadImageStream, releaseImage, err := extractReleaseImageStream(binDir, releaseImage, registryAuthfilePath)
+	releasePayloadImageStream, releaseImage, err := ExtractReleaseImageStream(binDir, releaseImage, registryAuthfilePath)
 	if err != nil {
 		return nil, errors.WithMessage(err, "couldn't extract release payload image stream")
 	}

pkg/test/extensions/util.go

@@ -8,6 +8,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	kapierrs "k8s.io/apimachinery/pkg/api/errors"
 	"os"
 	"os/exec"
 	"path"
@@ -159,7 +160,7 @@ func pullSpecToDirName(input string) string {
 	return filepath.Clean(safeName)
 }
 
-func determineReleasePayloadImage() (string, error) {
+func DetermineReleasePayloadImage() (string, error) {
 	var releaseImage string
 
 	// Highest priority override is EXTENSIONS_PAYLOAD_OVERRIDE
@@ -230,9 +231,9 @@ func createBinPath(path string) error {
 	return nil
 }
 
-// extractReleaseImageStream extracts image references from the given releaseImage and returns
+// ExtractReleaseImageStream extracts image references from the given releaseImage and returns
 // an ImageStream object with tags associated with image-references from that payload.
-func extractReleaseImageStream(extractPath, releaseImage string,
+func ExtractReleaseImageStream(extractPath, releaseImage string,
 	registryAuthFilePath string) (*imagev1.ImageStream, string, error) {
 
 	if _, err := os.Stat(path.Join(extractPath, "image-references")); err != nil {
@@ -269,3 +270,62 @@ func extractReleaseImageStream(extractPath, releaseImage string,
 
 	return is, releaseImage, nil
 }
+
+func DetermineRegistryAuthFilePath(tmpDir string, oc *util.CLI) (string, error) {
+	// To extract binaries bearing external tests, we must inspect the release
+	// payload under tests as well as extract content from component images
+	// referenced by that payload.
+	// openshift-tests is frequently run in the context of a CI job, within a pod.
+	// CI sets $RELEASE_IMAGE_LATEST to a pullspec for the release payload under test. This
+	// pull spec resolve to:
+	// 1. A build farm ci-op-* namespace / imagestream location (anonymous access permitted).
+	// 2. A quay.io/openshift-release-dev location (for tests against promoted ART payloads -- anonymous access permitted).
+	// 3. A registry.ci.openshift.org/ocp-<arch>/release:<tag> (request registry.ci.openshift.org token).
+	// Within the pod, we don't necessarily have a pull-secret for #3 OR the component images
+	// a payload references (which are private, unless in a ci-op-* imagestream).
+	// We try the following options:
+	// 1. If set, use the REGISTRY_AUTH_FILE environment variable to an auths file with
+	//    pull secrets capable of reading appropriate payload & component image
+	//    information.
+	// 2. If it exists, use a file /run/secrets/ci.openshift.io/cluster-profile/pull-secret
+	//    (conventional location for pull-secret information for CI cluster profile).
+	// 3. Use openshift-config secret/pull-secret from the cluster-under-test, if it exists
+	//    (Microshift does not).
+	// 4. Use unauthenticated access to the payload image and component images.
+	registryAuthFilePath := os.Getenv("REGISTRY_AUTH_FILE")
+
+	// if the environment variable is not set, extract the target cluster's
+	// platform pull secret.
+	if len(registryAuthFilePath) != 0 {
+		logrus.Infof("Using REGISTRY_AUTH_FILE environment variable: %v", registryAuthFilePath)
+	} else {
+
+		// See if the cluster-profile has stored a pull-secret at the conventional location.
+		ciProfilePullSecretPath := "/run/secrets/ci.openshift.io/cluster-profile/pull-secret"
+		_, err := os.Stat(ciProfilePullSecretPath)
+		if !os.IsNotExist(err) {
+			logrus.Infof("Detected %v; using cluster profile for image access", ciProfilePullSecretPath)
+			registryAuthFilePath = ciProfilePullSecretPath
+		} else {
+			// Inspect the cluster-under-test and read its cluster pull-secret dockerconfigjson value.
+			clusterPullSecret, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-config").Get(context.Background(), "pull-secret", metav1.GetOptions{})
+			if err != nil {
+				if kapierrs.IsNotFound(err) {
+					logrus.Warningf("Cluster has no openshift-config secret/pull-secret; falling back to unauthenticated image access")
+				} else {
+					return "", fmt.Errorf("unable to read ephemeral cluster pull secret: %w", err)
+				}
+			} else {
+				clusterDockerConfig := clusterPullSecret.Data[".dockerconfigjson"]
+				registryAuthFilePath = filepath.Join(tmpDir, ".dockerconfigjson")
+				err = os.WriteFile(registryAuthFilePath, clusterDockerConfig, 0600)
+				if err != nil {
+					return "", fmt.Errorf("unable to serialize target cluster pull-secret locally: %w", err)
+				}
+				logrus.Infof("Using target cluster pull-secrets for registry auth")
+			}
+		}
+	}
+
+	return registryAuthFilePath, nil
+}