feat(ws): Notebooks 2.0 // Backend // Backend can read data served by envtest

Signed-off-by: Yehudit Kerido <[email protected]>

Author: Yehudit Kerido

workspaces/backend/Makefile

@@ -86,6 +86,10 @@ build: fmt vet swag ## Build backend binary.
 run: fmt vet swag ## Run a backend from your host.
 	go run ./cmd/main.go --port=$(PORT)
 
+.PHONY: run-envtest
+run-envtest: fmt vet prepare-envtest-assets  ## Run envtest.
+	go run ./cmd/main.go --enable-envtest --port=$(PORT)
+
 # If you wish to build the manager image targeting other platforms you can use the --platform flag.
 # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
@@ -132,6 +136,11 @@ ENVTEST_VERSION ?= release-0.19
 GOLANGCI_LINT_VERSION ?= v1.61.0
 SWAGGER_VERSION ?= v1.16.4
 
+.PHONY: prepare-envtest-assets
+prepare-envtest-assets: envtest ## Download K8s control plane binaries directly into ./bin/k8s/
+	@echo ">>>> Downloading envtest Kubernetes control plane binaries to ./bin/k8s/..."
+	$(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=$(LOCALBIN)
+
 .PHONY: SWAGGER
 SWAGGER: $(SWAGGER)
 $(SWAGGER): $(LOCALBIN)

workspaces/backend/cmd/main.go

@@ -18,17 +18,22 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log/slog"
 	"os"
+	"path/filepath"
+	stdruntime "runtime"
 	"strconv"
 
+	"github.com/go-logr/logr"
+
 	ctrl "sigs.k8s.io/controller-runtime"
-	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	application "github.com/kubeflow/notebooks/workspaces/backend/api"
 	"github.com/kubeflow/notebooks/workspaces/backend/internal/auth"
 	"github.com/kubeflow/notebooks/workspaces/backend/internal/config"
 	"github.com/kubeflow/notebooks/workspaces/backend/internal/helper"
+	"github.com/kubeflow/notebooks/workspaces/backend/internal/k8sclientfactory"
 	"github.com/kubeflow/notebooks/workspaces/backend/internal/server"
 )
 
@@ -47,7 +52,7 @@ import (
 //	@consumes	application/json
 //	@produces	application/json
 
-func main() {
+func run() error {
 	// Define command line flags
 	cfg := &config.EnvConfig{}
 	flag.IntVar(&cfg.Port,
@@ -93,44 +98,59 @@ func main() {
 		"Key of request header containing user groups",
 	)
 
-	// Initialize the logger
-	logger := slog.New(slog.NewTextHandler(os.Stdout, nil))
+	var enableEnvTest bool
+	flag.BoolVar(&enableEnvTest,
+		"enable-envtest",
+		getEnvAsBool("ENABLE_ENVTEST", false),
+		"Enable envtest for local development without a real k8s cluster",
+	)
+	flag.Parse()
 
-	// Build the Kubernetes client configuration
-	kubeconfig, err := ctrl.GetConfig()
-	if err != nil {
-		logger.Error("failed to get Kubernetes config", "error", err)
-		os.Exit(1)
-	}
-	kubeconfig.QPS = float32(cfg.ClientQPS)
-	kubeconfig.Burst = cfg.ClientBurst
+	// Initialize the logger
+	slogTextHandler := slog.NewTextHandler(os.Stdout, nil)
+	logger := slog.New(slogTextHandler)
 
 	// Build the Kubernetes scheme
 	scheme, err := helper.BuildScheme()
 	if err != nil {
 		logger.Error("failed to build Kubernetes scheme", "error", err)
-		os.Exit(1)
+		return err
+	}
+
+	// Defining CRD's path
+	crdPath := os.Getenv("CRD_PATH")
+	if crdPath == "" {
+		_, currentFile, _, ok := stdruntime.Caller(0)
+		if !ok {
+			logger.Info("Failed to get current file path using stdruntime.Caller")
+		}
+		testFileDir := filepath.Dir(currentFile)
+		crdPath = filepath.Join(testFileDir, "..", "..", "controller", "config", "crd", "bases")
+		logger.Info("CRD_PATH not set, using guessed default", "path", crdPath)
 	}
 
-	// Create the controller manager
-	mgr, err := ctrl.NewManager(kubeconfig, ctrl.Options{
-		Scheme: scheme,
-		Metrics: metricsserver.Options{
-			BindAddress: "0", // disable metrics serving
-		},
-		HealthProbeBindAddress: "0", // disable health probe serving
-		LeaderElection:         false,
-	})
+	// ctx creates a context that listens for OS signals (e.g., SIGINT, SIGTERM) for graceful shutdown.
+	ctx := ctrl.SetupSignalHandler()
+
+	logrlogger := logr.FromSlogHandler(slogTextHandler)
+
+	// factory creates a new Kubernetes client factory, configured for envtest if enabled.
+	factory := k8sclientfactory.NewClientFactory(logrlogger, scheme, enableEnvTest, []string{crdPath}, cfg)
+
+	// Create the controller manager, build Kubernetes client configuration
+	// envtestCleanupFunc is a function to clean envtest if it was created, otherwise it's an empty function.
+	mgr, _, envtestCleanupFunc, err := factory.GetManagerAndConfig(ctx)
+	defer envtestCleanupFunc()
 	if err != nil {
-		logger.Error("unable to create manager", "error", err)
-		os.Exit(1)
+		logger.Error("Failed to get Kubernetes manager/config from factory", "error", err)
+		return err
 	}
 
 	// Create the request authenticator
 	reqAuthN, err := auth.NewRequestAuthenticator(cfg.UserIdHeader, cfg.UserIdPrefix, cfg.GroupsHeader)
 	if err != nil {
 		logger.Error("failed to create request authenticator", "error", err)
-		os.Exit(1)
+		return err
 	}
 
 	// Create the request authorizer
@@ -143,22 +163,30 @@ func main() {
 	app, err := application.NewApp(cfg, logger, mgr.GetClient(), mgr.GetScheme(), reqAuthN, reqAuthZ)
 	if err != nil {
 		logger.Error("failed to create app", "error", err)
-		os.Exit(1)
+		return err
 	}
 	svr, err := server.NewServer(app, logger)
 	if err != nil {
 		logger.Error("failed to create server", "error", err)
-		os.Exit(1)
+		return err
 	}
 	if err := svr.SetupWithManager(mgr); err != nil {
 		logger.Error("failed to setup server with manager", "error", err)
-		os.Exit(1)
+		return err
+	}
+
+	logger.Info("Starting manager...")
+	if err := mgr.Start(ctx); err != nil {
+		logger.Error("Problem running manager", "error", err)
+		return err
 	}
 
-	// Start the controller manager
-	logger.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
-		logger.Error("problem running manager", "error", err)
+	return nil
+}
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Application run failed: %v\n", err)
 		os.Exit(1)
 	}
 }

workspaces/backend/go.mod

@@ -33,12 +33,10 @@ require (
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
-	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
-	github.com/go-openapi/spec v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
@@ -55,7 +53,6 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
@@ -67,7 +64,6 @@ require (
 	github.com/spf13/cobra v1.8.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stoewer/go-strcase v1.2.0 // indirect
-	github.com/swaggo/files/v2 v2.0.2 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect
 	go.opentelemetry.io/otel v1.28.0 // indirect
@@ -87,7 +83,6 @@ require (
 	golang.org/x/term v0.29.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.30.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect
@@ -103,5 +98,13 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.30.3 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.4.0 // indirect
+)
+
+require (
+	github.com/go-logr/logr v1.4.2
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/swaggo/files/v2 v2.0.2 // indirect
+	golang.org/x/tools v0.30.0 // indirect
+	sigs.k8s.io/yaml v1.4.0
 )

workspaces/backend/internal/k8sclientfactory/client_factory.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2024.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package k8sclientfactory
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/go-logr/logr"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
+
+	"github.com/kubeflow/notebooks/workspaces/backend/internal/config"
+
+	"k8s.io/client-go/rest"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kubeflow/notebooks/workspaces/backend/localdev"
+)
+
+// ClientFactory responsible for providing a Kubernetes client and manager
+type ClientFactory struct {
+	useEnvtest  bool
+	crdPaths    []string
+	logger      logr.Logger
+	scheme      *runtime.Scheme
+	clientQPS   float64
+	clientBurst int
+}
+
+// NewClientFactory creates a new factory
+func NewClientFactory(
+	logger logr.Logger,
+	scheme *runtime.Scheme,
+	useEnvtest bool,
+	crdPaths []string,
+	appCfg *config.EnvConfig,
+) *ClientFactory {
+	return &ClientFactory{
+		useEnvtest:  useEnvtest,
+		crdPaths:    crdPaths,
+		logger:      logger.WithName("k8s-client-factory"),
+		scheme:      scheme,
+		clientQPS:   appCfg.ClientQPS,
+		clientBurst: appCfg.ClientBurst,
+	}
+}
+
+// GetManagerAndConfig returns a configured Kubernetes manager and its rest.Config
+// It also returns a cleanup function for envtest if it was started.
+func (f *ClientFactory) GetManagerAndConfig(ctx context.Context) (ctrl.Manager, *rest.Config, func(), error) {
+	var mgr ctrl.Manager
+	var cfg *rest.Config
+	var err error
+	var cleanupFunc func() = func() {} // No-op cleanup by default
+
+	if f.useEnvtest {
+		f.logger.Info("Using envtest mode: setting up local Kubernetes environment...")
+		var testEnvInstance *envtest.Environment
+
+		cfg, mgr, testEnvInstance, err = localdev.StartLocalDevEnvironment(ctx, f.crdPaths, f.scheme)
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("could not start local dev environment: %w", err)
+		}
+		f.logger.Info("Local dev K8s API (envtest) is ready.", "host", cfg.Host)
+
+		if testEnvInstance != nil {
+			cleanupFunc = func() {
+				f.logger.Info("Stopping envtest environment...")
+				if err := testEnvInstance.Stop(); err != nil {
+					f.logger.Error(err, "Failed to stop envtest environment")
+				}
+			}
+		} else {
+			err = errors.New("StartLocalDevEnvironment returned successfully but with a nil testEnv instance, cleanup is not possible")
+			f.logger.Error(err, "invalid return state from localdev setup")
+			return nil, nil, nil, err
+		}
+	} else {
+		f.logger.Info("Using real cluster mode: connecting to existing Kubernetes cluster...")
+		cfg, err = ctrl.GetConfig()
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("failed to get Kubernetes config: %w", err)
+		}
+		f.logger.Info("Successfully connected to existing Kubernetes cluster.")
+
+		cfg.QPS = float32(f.clientQPS)
+		cfg.Burst = f.clientBurst
+		mgr, err = ctrl.NewManager(cfg, ctrl.Options{
+			Scheme: f.scheme,
+			Metrics: metricsserver.Options{
+				BindAddress: "0", // disable metrics serving
+			},
+			HealthProbeBindAddress: "0", // disable health probe serving
+			LeaderElection:         false,
+		})
+		if err != nil {
+			return nil, nil, nil, fmt.Errorf("unable to create manager for real cluster: %w", err)
+		}
+		f.logger.Info("Successfully configured manager for existing Kubernetes cluster.")
+	}
+	return mgr, cfg, cleanupFunc, nil
+}
+
+// GetClient returns just the client.Client (useful if manager lifecycle is handled elsewhere or already started)
+func (f *ClientFactory) GetClient(ctx context.Context) (client.Client, func(), error) {
+	mgr, _, cleanup, err := f.GetManagerAndConfig(ctx)
+	if err != nil {
+		if cleanup != nil {
+			f.logger.Info("Calling cleanup function due to error during manager/config retrieval", "error", err)
+			cleanup()
+		}
+		return nil, cleanup, err
+	}
+	return mgr.GetClient(), cleanup, nil
+}