Merge pull request #30064 from smg247/remove-annotations

TRT-2071: Remove annotations and replace with environment selector based test skipping and disablement

Author: openshift-merge-bot[bot]

README.md

@@ -37,34 +37,29 @@ will switch to vendoring origin-specific branches (e.g
 carries that need to be considered in the context of
 `openshift/kubernetes` rebases.
 
-### Test annotation rules
+### Test exclusion rules
 
-Test annotation rules are used to label e2e tests so that they can be
-filtered or skipped. For example, rules can be defined that match kube
-e2e tests that are known to be incompatible with openshift and label
-those tests to be skipped.
+Test exclusion is now handled through environmental selector based filtering
+rather than test annotations. Environmental selectors allow tests to be
+filtered or skipped based on the cluster environment and configuration.
+For example, selectors can be defined that match kube e2e tests that are
+known to be incompatible with specific OpenShift configurations and exclude
+those tests from running.
 
-Maintenance of test annotation rules is split between the
+Maintenance of test exclusion rules is split between the
 `openshift/kubernetes` and `origin` repos to ensure that PRs proposed
 to `openshift/kubernetes` can be validated against the set of kube e2e
-tests known to be compatible with openshift.
+tests known to be compatible with OpenShift.
 
-Test annotation rules for kubernetes e2e tests are maintained in:
+Test exclusion rules for kubernetes e2e tests are maintained in: https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext:
+* [environment_selectors.go](https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext/environment_selectors.go)
+* [disabled_tests.go](https://github.com/openshift/kubernetes/blob/master/openshift-hack/cmd/k8s-tests-ext/disabled_tests.go)
 
-https://github.com/openshift/kubernetes/blob/master/openshift-hack/e2e/annotate/rules.go
+Test exclusion rules for openshift e2e tests are maintained in: https://github.com/openshift/origin/blob/main/pkg/test/extensions:
+* [environment_selectors.go](https://github.com/openshift/origin/blob/main/pkg/test/extensions/environment_selectors.go)
+* [disabled_tests.go](https://github.com/openshift/origin/blob/main/pkg/test/extensions/disabled_tests.go)
 
-Test annotation rules for openshift e2e tests are maintained in:
-
-https://github.com/openshift/origin/blob/main/test/extended/util/annotate/rules.go
-
-Origin vendors the kube rules and applies both the kube and openshift
-rules to the set of tests included in the `openshift-tests` binary.
-
-In order to update test annotation rules for kube e2e tests, it will
-be necessary to:
-
- - Update `rules.go` in `openshift/kubernetes`
- - Bump the version of `openshift/kubernetes` vendored in origin
+To update test exclusion rules for kube e2e tests, update the environmental selectors in `openshift/kubernetes`. For OpenShift e2e tests, update the selectors in `origin`.
 
 ### Vendoring from `openshift/kubernetes`
 

cmd/openshift-tests/openshift-tests.go

@@ -28,7 +28,6 @@ import (
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/monitor"
 	run_monitor "github.com/openshift/origin/pkg/cmd/openshift-tests/monitor/run"
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/monitor/timeline"
-	"github.com/openshift/origin/pkg/cmd/openshift-tests/render"
 	risk_analysis "github.com/openshift/origin/pkg/cmd/openshift-tests/risk-analysis"
 	"github.com/openshift/origin/pkg/cmd/openshift-tests/run"
 	run_disruption "github.com/openshift/origin/pkg/cmd/openshift-tests/run-disruption"
@@ -106,7 +105,6 @@ func main() {
 		timeline.NewTimelineCommand(ioStreams),
 		run_disruption.NewRunInClusterDisruptionMonitorCommand(ioStreams),
 		collectdiskcertificates.NewRunCollectDiskCertificatesCommand(ioStreams),
-		render.NewRenderCommand(ioStreams),
 		versioncmd.NewVersionCommand(ioStreams),
 	)
 

hack/update-generated.sh

@@ -7,11 +7,6 @@ source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 # https://github.com/kubernetes/kubernetes/pull/109541
 ENABLE_STORAGE_GCE_PD_DRIVER=yes go generate -mod vendor ./test/extended
 
-# the test names are now contained in test/extended/util/annotate/generated/zz_generated.annotations.go
-# A txt file would be more extensible, but Maciej stacked the debt here and he's the same person who will have pay it down
-# as we add more images with more tests, so this seems acceptable.
-go run -mod vendor ./cmd/openshift-tests render test-report --output-dir=./zz_generated.manifests
-
 # Update mirror mapping from upstream to quay
 # By default, "openshift-tests images" lists images from external binaries. However, we force
 # this script to list images from built-in tests in order to avoid requiring an OCP release image.

hack/verify-generated.sh

@@ -11,8 +11,6 @@ trap "cleanup" EXIT
 
 os::test::junit::declare_suite_start "verify/generated"
 os::cmd::expect_success "${OS_ROOT}/hack/update-generated.sh"
-os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/test/extended/util/annotate/generated/"
 os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/test/extended/util/image/zz_generated.txt"
-os::cmd::expect_success "git diff --exit-code ${OS_ROOT}/zz_generated.manifests"
 
 os::test::junit::declare_suite_end

images/tests/Dockerfile.rhel

@@ -7,8 +7,6 @@ RUN make; \
 
 FROM registry.ci.openshift.org/ocp/4.20:tools
 COPY --from=builder /tmp/build/openshift-tests /usr/bin/
-RUN mkdir -p /manifests
-COPY --from=builder /go/src/github.com/openshift/origin/zz_generated.manifests/* /manifests
 
 RUN PACKAGES="git gzip util-linux" && \
     if [ $HOSTTYPE = x86_64 ]; then PACKAGES="$PACKAGES python3-cinderclient"; fi && \

pkg/cmd/openshift-tests/render/render_command.go

@@ -16,11 +16,10 @@ import (
 	"syscall"
 	"time"
 
-	originVersion "github.com/openshift/origin/pkg/version"
-
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension/extensiontests"
 	g "github.com/openshift-eng/openshift-tests-extension/pkg/ginkgo"
+	originVersion "github.com/openshift/origin/pkg/version"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/mod/semver"
@@ -34,7 +33,6 @@ import (
 	"github.com/openshift/origin/pkg/clioptions/imagesetup"
 	"github.com/openshift/origin/pkg/clioptions/upgradeoptions"
 	exutil "github.com/openshift/origin/test/extended/util"
-	origingenerated "github.com/openshift/origin/test/extended/util/annotate/generated"
 	"github.com/openshift/origin/test/extended/util/image"
 )
 
@@ -78,24 +76,26 @@ func InitializeOpenShiftTestsExtensionFramework() (*extension.Registry, *extensi
 		return nil, nil, fmt.Errorf("failed to build extension test specs: %w", err)
 	}
 
-	// Apply annotations to test names
+	// Apply annotations to test names only for upstream tests
 	specs.Walk(func(spec *extensiontests.ExtensionTestSpec) {
-		// we need to ensure the default path always annotates both
-		// origin and k8s tests accordingly, since each of these
-		// currently have their own annotations which are not
-		// merged anywhere else but applied here
-		if append, ok := origingenerated.Annotations[spec.Name]; ok {
-			spec.Name += append
-		}
 		if append, ok := k8sgenerated.Annotations[spec.Name]; ok {
 			spec.Name += append
 		}
 	})
 
+	klog.Infof("Found %d test specs", len(specs))
 	// Filter out kube tests, vendor filtering isn't working within origin
 	specs = specs.Select(func(spec *extensiontests.ExtensionTestSpec) bool {
 		return !strings.Contains(spec.Name, "[Suite:k8s")
 	})
+	klog.Infof("%d test specs remain, after filtering out k8s", len(specs))
+
+	// Filter out tests that are always disabled based on name matching
+	specs = filterOutDisabledSpecs(specs)
+	// Add environment selectors that include or exclude tests in specific environments
+	addEnvironmentSelectors(specs)
+	addLabelsToSpecs(specs)
+	appendSuiteNames(specs)
 
 	specs.AddBeforeAll(func() {
 		config, err := clusterdiscovery.DecodeProvider(os.Getenv("TEST_PROVIDER"), false, false, nil)