add testing & support feature-gate in helm chart

Author: cchen777

helm-chart/kuberay-operator/templates/_helpers.tpl

@@ -54,3 +54,18 @@ Create the name of the service account to use
     {{ default "default" .Values.serviceAccount.name }}
 {{- end -}}
 {{- end -}}
+
+
+{{/*
+FeatureGates
+*/}}
+{{- define "kuberay.featureGates" -}}
+{{- $features := "" }}
+{{- range .Values.featureGates }}
+  {{- $str := printf "%s=%t," .name .enabled }}
+  {{- $features = print $features $str }}
+{{- end }}
+{{- with .Values.featureGates }}
+--feature-gates={{ $features | trimSuffix "," }}
+{{- end }}
+{{- end }}

helm-chart/kuberay-operator/templates/deployment.yaml

@@ -56,6 +56,7 @@ spec:
             - /manager
           args:
             {{- $argList := list -}}
+            {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}}
             {{- if .Values.batchScheduler.enabled -}}
             {{- $argList = append $argList "--enable-batch-scheduler" -}}
             {{- end -}}

helm-chart/kuberay-operator/values.yaml

@@ -58,6 +58,11 @@ readinessProbe:
 batchScheduler:
   enabled: false
 
+featureGates:
+  - name: RayClusterStatusConditions
+    enabled: false
+
+
 # Set up `securityContext` to improve Pod security.
 # See https://github.com/ray-project/kuberay/blob/master/docs/guidance/pod-security.md for further guidance.
 podSecurityContext: {}

ray-operator/controllers/ray/raycluster_controller_unit_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ray-project/kuberay/ray-operator/controllers/ray/common"
 	"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
 	"github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned/scheme"
+	"github.com/ray-project/kuberay/ray-operator/pkg/features"
 
 	. "github.com/onsi/ginkgo/v2"
 	"github.com/stretchr/testify/assert"
@@ -33,6 +34,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -1664,6 +1666,12 @@ func TestCalculateStatus(t *testing.T) {
 		Status: corev1.PodStatus{
 			PodIP: headNodeIP,
 			Phase: corev1.PodRunning,
+			Conditions: []corev1.PodCondition{
+				{
+					Type:   corev1.PodReady,
+					Status: corev1.ConditionTrue,
+				},
+			},
 		},
 	}
 	runtimeObjects := []runtime.Object{headPod, headService}
@@ -1687,6 +1695,47 @@ func TestCalculateStatus(t *testing.T) {
 	assert.Equal(t, headService.Name, newInstance.Status.Head.ServiceName)
 	assert.NotNil(t, newInstance.Status.StateTransitionTimes, "Cluster state transition timestamp should be created")
 	assert.Equal(t, newInstance.Status.LastUpdateTime, newInstance.Status.StateTransitionTimes[rayv1.Ready])
+
+	// Test empty conditions with feature gate disabled
+	newInstance, _ = r.calculateStatus(ctx, testRayCluster, nil)
+	assert.Empty(t, newInstance.Status.Conditions)
+
+	// enable feature gate for the following tests
+	defer features.SetFeatureGateDuringTest(t, features.RayClusterStatusConditions, true)()
+
+	// Test CheckRayHeadRunningAndReady with head pod running and ready
+	newInstance, _ = r.calculateStatus(ctx, testRayCluster, nil)
+	assert.True(t, meta.IsStatusConditionPresentAndEqual(newInstance.Status.Conditions, string(rayv1.HeadReady), metav1.ConditionTrue))
+	condition := meta.FindStatusCondition(newInstance.Status.Conditions, string(rayv1.HeadReady))
+	assert.Equal(t, "HeadPodRunningAndReady", condition.Reason)
+	assert.Equal(t, "Head pod is running and ready", condition.Message)
+
+	// Test CheckRayHeadRunningAndReady with head pod not ready
+	headPod.Status.Conditions = []corev1.PodCondition{
+		{
+			Type:   corev1.PodReady,
+			Status: corev1.ConditionFalse,
+		},
+	}
+	runtimeObjects = []runtime.Object{headPod, headService}
+	fakeClient = clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(runtimeObjects...).Build()
+	r.Client = fakeClient
+	newInstance, _ = r.calculateStatus(ctx, testRayCluster, nil)
+	assert.True(t, meta.IsStatusConditionPresentAndEqual(newInstance.Status.Conditions, string(rayv1.HeadReady), metav1.ConditionFalse))
+	condition = meta.FindStatusCondition(newInstance.Status.Conditions, string(rayv1.HeadReady))
+	assert.Equal(t, "HeadPodNotReady", condition.Reason)
+	assert.Equal(t, "Head pod is not ready", condition.Message)
+
+	// Test CheckRayHeadRunningAndReady with head pod not running
+	headPod.Status.Phase = corev1.PodFailed
+	runtimeObjects = []runtime.Object{headPod, headService}
+	fakeClient = clientFake.NewClientBuilder().WithScheme(newScheme).WithRuntimeObjects(runtimeObjects...).Build()
+	r.Client = fakeClient
+	newInstance, _ = r.calculateStatus(ctx, testRayCluster, nil)
+	assert.True(t, meta.IsStatusConditionPresentAndEqual(newInstance.Status.Conditions, string(rayv1.HeadReady), metav1.ConditionFalse))
+	condition = meta.FindStatusCondition(newInstance.Status.Conditions, string(rayv1.HeadReady))
+	assert.Equal(t, "HeadPodNotRunning", condition.Reason)
+	assert.Equal(t, "Head pod is not running", condition.Message)
 }
 
 func TestStateTransitionTimes_NoStateChange(t *testing.T) {

ray-operator/controllers/ray/utils/util.go

@@ -94,7 +94,7 @@ func CheckRayHeadRunningAndReady(ctx context.Context, pods corev1.PodList) (bool
 		log.Info("No head pod found in the pod list")
 		return false, "HeadPodNotFound", "No head pod found in the pod list"
 	}
-	log.Info("Head pod is running and ready !!!")
+	log.Info("Head pod is running and ready")
 	return true, "HeadPodRunningAndReady", "Head pod is running and ready"
 }
 

ray-operator/controllers/ray/utils/util_test.go

@@ -201,6 +201,31 @@ func createSomePodWithCondition(typ corev1.PodConditionType, status corev1.Condi
 	}
 }
 
+func createRayHeadPodWithPhaseAndCOndition(phase corev1.PodPhase, typ corev1.PodConditionType, status corev1.ConditionStatus) (pod *corev1.Pod) {
+	return &corev1.Pod{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "Pod",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "raycluster-sample-head",
+			Namespace: "default",
+			Labels: map[string]string{
+				"ray.io/node-type": string(rayv1.HeadNode),
+			},
+		},
+		Status: corev1.PodStatus{
+			Phase: phase,
+			Conditions: []corev1.PodCondition{
+				{
+					Type:   typ,
+					Status: status,
+				},
+			},
+		},
+	}
+}
+
 func TestGetHeadGroupServiceAccountName(t *testing.T) {
 	tests := map[string]struct {
 		input *rayv1.RayCluster
@@ -523,3 +548,42 @@ env_vars:
 		})
 	}
 }
+
+func TestCheckRayHeadRunningAndReady(t *testing.T) {
+	tests := map[string]struct {
+		pods     corev1.PodList
+		expected bool
+	}{
+		"should return true if Ray head pod is running and ready": {
+			pods: corev1.PodList{
+				Items: []corev1.Pod{
+					*createRayHeadPodWithPhaseAndCOndition(corev1.PodRunning, corev1.PodReady, corev1.ConditionTrue),
+				},
+			},
+			expected: true,
+		},
+		"should return false if Ray head pod is not running": {
+			pods: corev1.PodList{
+				Items: []corev1.Pod{
+					*createRayHeadPodWithPhaseAndCOndition(corev1.PodPending, corev1.PodReady, corev1.ConditionFalse),
+				},
+			},
+			expected: false,
+		},
+		"should return false if Ray head pod is not ready": {
+			pods: corev1.PodList{
+				Items: []corev1.Pod{
+					*createRayHeadPodWithPhaseAndCOndition(corev1.PodRunning, corev1.PodReady, corev1.ConditionFalse),
+				},
+			},
+			expected: false,
+		},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			isRunning, _, _ := CheckRayHeadRunningAndReady(context.Background(), tc.pods)
+			assert.Equal(t, tc.expected, isRunning)
+		})
+	}
+}