test (refactor): adding POC Sdk Compile tests

.gitignore

@@ -93,3 +93,8 @@ kubeconfig_dev-pipelines-api
 backend/Dockerfile.driver-debug
 
 backend/src/crd/kubernetes/bin
+
+**/allure-*
+**/*.html
+reports/
+logs/

.pre-commit-config.yaml

@@ -38,8 +38,8 @@ repos:
       - id: isort
         name: isort
         entry: isort --profile google
-  - repo: https://github.com/pre-commit/mirrors-yapf
-    rev: "v0.32.0"
+  - repo: https://github.com/google/yapf
+    rev: "v0.43.0"
     hooks:
       - id: yapf
   - repo: https://github.com/pycqa/docformatter

backend/test/v2/api/constants/severity.go

@@ -0,0 +1,19 @@
+package api
+
+type Severity int
+
+const (
+	S1 Severity = iota
+	S2
+	S3
+)
+
+var severityName = map[Severity]string{
+	S1: "CRITICAL",
+	S2: "MEDIUM",
+	S3: "TRIVIAL",
+}
+
+func (s Severity) String() string {
+	return severityName[s]
+}

backend/test/v2/api/constants/test_type.go

@@ -0,0 +1,19 @@
+package api
+
+type TestType int
+
+const (
+	SMOKE TestType = iota
+	CRITICAL_ONLY
+	FULL_SUITE
+)
+
+var testTypeName = map[TestType]string{
+	SMOKE:         "SMOKE",
+	CRITICAL_ONLY: "CRITICAL_ONLY",
+	FULL_SUITE:    "FULL_SUITE",
+}
+
+func (s TestType) String() string {
+	return testTypeName[s]
+}

backend/test/v2/api/flags.go

@@ -0,0 +1,44 @@
+// Copyright 2018-2023 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package api
+
+import (
+	"flag"
+	"time"
+)
+
+var (
+	namespace                   = flag.String("namespace", "kubeflow", "The namespace ml pipeline deployed to")
+	initializeTimeout           = flag.Duration("initializeTimeout", 2*time.Minute, "Duration to wait for test initialization")
+	runIntegrationTests         = flag.Bool("runIntegrationTests", true, "Whether to also run integration tests that call the service")
+	runK8sStoreIntegrationTests = flag.Bool("runK8sStoreIntegrationTests", false, "Whether to run integration tests with K8s Native API Support")
+	runUpgradeTests             = flag.Bool("runUpgradeTests", false, "Whether to run upgrade tests")
+	useProxy                    = flag.Bool("useProxy", false, "Whether to run the proxy tests")
+	cacheEnabled                = flag.Bool("cacheEnabled", true, "Whether cache is enabled tests")
+)
+
+/**
+ * Differences in dev mode:
+ * 1. Resources are not cleaned up when a test finishes, so that developer can debug manually.
+ * 2. One step that doesn't work locally is skipped.
+ */
+var isDevMode = flag.Bool("isDevMode", false, "Dev mode helps local development of integration tests")
+
+var isDebugMode = flag.Bool("isDebugMode", false, "Whether to enable debug mode. Debug mode will log more diagnostics messages.")
+
+var (
+	isKubeflowMode    = flag.Bool("isKubeflowMode", false, "Runs tests in full Kubeflow mode")
+	resourceNamespace = flag.String("resourceNamespace", "", "The namespace that will store the test resources in Kubeflow mode")
+)

backend/test/v2/api/integration_suite_test.go

@@ -0,0 +1,102 @@
+package api
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"testing"
+
+	api_server "github.com/kubeflow/pipelines/backend/src/common/client/api_server/v2"
+	"github.com/kubeflow/pipelines/backend/test/v2/api/logger"
+	test "github.com/kubeflow/pipelines/backend/test/v2/api/utils"
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/ginkgo/v2/types"
+	. "github.com/onsi/gomega"
+)
+
+var pipelineUploadClient *api_server.PipelineUploadClient
+var pipelineClient *api_server.PipelineClient
+var testLogsDirectory = "logs"
+var testReportDirectory = "reports"
+var junitReportFilename = "junit.xml"
+var jsonReportFilename = "report.json"
+
+var _ = BeforeSuite(func() {
+	err := os.MkdirAll(testLogsDirectory, 0755)
+	if err != nil {
+		logger.Log("Error creating logs directory: %s", testLogsDirectory)
+		return
+	}
+	err = os.MkdirAll(testReportDirectory, 0755)
+	if err != nil {
+		logger.Log("Error creating reports directory: %s", testReportDirectory)
+		return
+	}
+	var newPipelineUploadClient func() (*api_server.PipelineUploadClient, error)
+	var newPipelineClient func() (*api_server.PipelineClient, error)
+
+	if *isKubeflowMode {
+		newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
+			return api_server.NewKubeflowInClusterPipelineUploadClient(*namespace, *isDebugMode)
+		}
+		newPipelineClient = func() (*api_server.PipelineClient, error) {
+			return api_server.NewKubeflowInClusterPipelineClient(*namespace, *isDebugMode)
+		}
+	} else {
+		clientConfig := test.GetClientConfig(*namespace)
+
+		newPipelineUploadClient = func() (*api_server.PipelineUploadClient, error) {
+			return api_server.NewPipelineUploadClient(clientConfig, *isDebugMode)
+		}
+		newPipelineClient = func() (*api_server.PipelineClient, error) {
+			return api_server.NewPipelineClient(clientConfig, *isDebugMode)
+		}
+	}
+
+	pipelineUploadClient, err = newPipelineUploadClient()
+	if err != nil {
+		logger.Log("Failed to get pipeline upload client. Error: %s", err.Error())
+	}
+	pipelineClient, err = newPipelineClient()
+	if err != nil {
+		logger.Log("Failed to get pipeline client. Error: %s", err.Error())
+	}
+})
+
+var _ = ReportAfterEach(func(specReport types.SpecReport) {
+	if specReport.Failed() {
+		logger.Log("Test failed... Capture pod logs if you want to")
+		AddReportEntry("Pod Log", "Pod Logs")
+		AddReportEntry("Test Log", specReport.CapturedGinkgoWriterOutput)
+		writeLogFile(specReport)
+	} else {
+		log.Printf("Test passed")
+	}
+})
+
+func writeLogFile(specReport types.SpecReport) {
+	stdOutput := specReport.CapturedGinkgoWriterOutput
+	testName := GinkgoT().Name()
+	testLogFile := filepath.Join(testLogsDirectory, testName+".log")
+	logFile, err := os.Create(testLogFile)
+	if err != nil {
+		logger.Log("Failed to create log file due to: %s", err.Error())
+	}
+	_, err = logFile.Write([]byte(stdOutput))
+	if err != nil {
+		logger.Log("Failed to write to the log file, due to: %s", err.Error())
+	}
+	logFile.Close()
+}
+
+func TestAPIs(t *testing.T) {
+	RegisterFailHandler(Fail)
+	suiteConfig, reporterConfig := GinkgoConfiguration()
+	suiteConfig.EmitSpecProgress = true
+	suiteConfig.FailFast = false
+	reporterConfig.GithubOutput = true
+	reporterConfig.ShowNodeEvents = true
+	reporterConfig.JUnitReport = filepath.Join(testReportDirectory, junitReportFilename)
+	reporterConfig.JSONReport = filepath.Join(testReportDirectory, jsonReportFilename)
+	RunSpecs(t, "API Tests Suite", suiteConfig, reporterConfig)
+}

backend/test/v2/api/logger/logger.go

@@ -0,0 +1,11 @@
+package logger
+
+import (
+	"fmt"
+
+	"github.com/onsi/ginkgo/v2"
+)
+
+func Log(s string, arguments ...any) {
+	ginkgo.GinkgoWriter.Println(fmt.Sprintf(s, arguments))
+}

backend/test/v2/api/matcher/custom_matcher.go

@@ -0,0 +1,38 @@
+// Package api
+// Copyright 2018-2023 The Kubeflow Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package api
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/onsi/gomega"
+)
+
+// MatchMaps - Iterate over 2 maps and compare if they are same or not
+func MatchMaps(actual interface{}, expected interface{}, mapType string) {
+	expectedMap, _ := expected.(map[any]interface{})
+	actualMap, _ := actual.(map[any]interface{})
+	for key, value := range expectedMap {
+		if reflect.TypeOf(value).Kind() == reflect.Map {
+			expectedMapFromValue, _ := value.(map[any]interface{})
+			actualMapFromValue, _ := actualMap[key].(map[any]interface{})
+			MatchMaps(expectedMapFromValue, actualMapFromValue, mapType)
+		}
+		actualStringValue := fmt.Sprintf("%v", actualMap[key])
+		expectedStringValue := fmt.Sprintf("%v", value)
+		gomega.Expect(actualStringValue).To(gomega.Equal(expectedStringValue), fmt.Sprintf("'%s' key's value not matching for type %s", key, mapType))
+	}
+}

backend/test/v2/api/matcher/pipeline_matcher.go

@@ -0,0 +1,39 @@
+package api
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	model "github.com/kubeflow/pipelines/backend/api/v2beta1/go_http_client/pipeline_upload_model"
+	. "github.com/onsi/gomega"
+)
+
+func MatchPipelines(actual *model.V2beta1Pipeline, expected *model.V2beta1Pipeline) {
+	Expect(actual.PipelineID).To(Not(BeEmpty()), "Pipeline ID is empty")
+	actualTime := time.Time(actual.CreatedAt).UTC()
+	expectedTime := time.Time(expected.CreatedAt).UTC()
+	Expect(actualTime.After(expectedTime)).To(BeTrue(), "Actual Pipeline creation time is not as expected")
+	Expect(actual.DisplayName).To(Equal(expected.DisplayName), "Pipeline name not matching")
+	Expect(actual.Namespace).To(Equal(expected.Namespace), "Pipeline Namespace not matching")
+	Expect(actual.Description).To(Equal(expected.Description), "Pipeline Description not matching")
+
+}
+
+func MatchPipelineVersions(actual *model.V2beta1PipelineVersion, expected *model.V2beta1PipelineVersion) {
+	Expect(actual.PipelineVersionID).To(Not(Equal(expected.PipelineVersionID)), "Pipeline Version ID is empty")
+	actualTime := time.Time(actual.CreatedAt).UTC()
+	expectedTime := time.Time(expected.CreatedAt).UTC()
+	Expect(actualTime.After(expectedTime)).To(BeTrue(), "Actual Pipeline Version creation time is not as expected")
+	Expect(actual.DisplayName).To(Equal(expected.DisplayName), "Pipeline Display Name not matching")
+	Expect(actual.Description).To(Equal(expected.Description), "Pipeline Description not matching")
+	MatchMaps(actual.PipelineSpec, expected.PipelineSpec, "Pipeline Spec")
+}
+
+func MatchPipelineVersionSpec(actual any, expected map[string]interface{}) {
+	if reflect.TypeOf(actual).Kind() == reflect.Map {
+		for key, value := range expected {
+			Expect(actual).To(HaveKeyWithValue(key, value), fmt.Sprintf("%s value not matching", key))
+		}
+	}
+}