Merge branch 'kubeflow:master' into feature/kubeflow-sdk-integrate

Signed-off-by: Prateek Bala <[email protected]>

Author: Prateekbala

.github/actions/deploy/action.yml

@@ -27,6 +27,10 @@ inputs:
     description: "If KFP should be deployed in multi-user mode"
     required: false
     default: 'false'
+  artifact_proxy:
+    description: "Enables artifact proxy"
+    required: false
+    default: 'false'
   storage_backend:
     description: "Storage backend to use (minio or seaweedfs)"
     required: false
@@ -94,6 +98,11 @@ runs:
           ARGS="${ARGS} --multi-user"
         fi
 
+        if [ "${{ inputs.artifact_proxy }}" = "true" ]; then
+          echo "Enabling artifact proxy"
+          ARGS="${ARGS} --artifact-proxy"
+        fi
+
         if [ "${{ inputs.storage_backend }}" != "seaweedfs" ] && [ -n "${{ inputs.storage_backend }}" ]; then
           echo "Deploying with artifact storage ${{ inputs.storage_backend }}"
           ARGS="${ARGS} --storage ${{ inputs.storage_backend }}"

.github/resources/manifests/multiuser/artifact-proxy/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../default
+
+patches:
+- target:
+    kind: ConfigMap
+    name: pipeline-install-config
+  patch: |
+    - op: add
+      path: /data/ARTIFACTS_PROXY_ENABLED
+      value: "true"
+- target:
+    kind: Deployment
+    name: kubeflow-pipelines-profile-controller
+  patch: |
+    - op: add
+      path: /spec/template/spec/containers/0/env/-
+      value:
+        name: DISABLE_ISTIO_SIDECAR 
+        value: "true" # Dangerous and only for CI/CD, not production usage
+

.github/resources/scripts/deploy-kfp.sh

@@ -28,6 +28,7 @@ TEST_MANIFESTS=".github/resources/manifests"
 PIPELINES_STORE="database"
 USE_PROXY=false
 CACHE_DISABLED=false
+ARTIFACT_PROXY_ENABLED=false
 MULTI_USER=false
 STORAGE_BACKEND="seaweedfs"
 AWF_VERSION=""
@@ -55,6 +56,10 @@ while [ "$#" -gt 0 ]; do
       MULTI_USER=true
       shift
       ;;
+    --artifact-proxy)
+      ARTIFACT_PROXY_ENABLED=true
+      shift
+      ;;
     --storage)
       STORAGE_BACKEND="$2"
       shift 2
@@ -86,11 +91,6 @@ if [ "${MULTI_USER}" == "true" ] && [ "${USE_PROXY}" == "true" ]; then
   exit 1
 fi
 
-if [ "${STORAGE_BACKEND}" != "minio" ] && [ "${STORAGE_BACKEND}" != "seaweedfs" ]; then
-  echo "ERROR: Storage backend must be either 'minio' or 'seaweedfs'."
-  exit 1
-fi
-
 if [ -n "${AWF_VERSION}"  ]; then
   echo "NOTE: Argo version ${AWF_VERSION} specified, updating Argo Workflow manifests..."
   echo "${AWF_VERSION}" > third_party/argo/VERSION
@@ -134,15 +134,6 @@ if [ "${MULTI_USER}" == "true" ]; then
   echo "Installing Profile Controller Resources..."
   kubectl apply -k https://github.com/kubeflow/manifests/applications/profiles/upstream/overlays/kubeflow?ref=master
   kubectl -n kubeflow wait --for=condition=Ready pods -l kustomize.component=profiles --timeout 180s
-
-  echo "Creating KF Profile..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/test-profiles.yaml
-
-  echo "Applying kubeflow-edit ClusterRole with proper aggregation..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml
-
-  echo "Applying network policy to allow user namespace access to kubeflow services..."
-  kubectl apply -f test_data/kubernetes/seaweedfs/allow-user-namespace-access.yaml
 fi
 
 # Manifests will be deployed according to the flag provided
@@ -176,17 +167,20 @@ elif [ "${MULTI_USER}" == "false" ] && [ "${PIPELINES_STORE}" == "kubernetes" ];
   fi
 elif [ "${MULTI_USER}" == "true" ]; then
   TEST_MANIFESTS="${TEST_MANIFESTS}/multiuser"
-  if [ "${STORAGE_BACKEND}" == "minio" ]; then
+  if $ARTIFACT_PROXY_ENABLED && [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/artifact-proxy"
+  elif [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/minio"
-  elif $CACHE_DISABLED; then
-    TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled"
   elif $CACHE_DISABLED && [ "${STORAGE_BACKEND}" == "minio" ]; then
     TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled-minio"
+  elif $CACHE_DISABLED; then
+    TEST_MANIFESTS="${TEST_MANIFESTS}/cache-disabled"
   else
     TEST_MANIFESTS="${TEST_MANIFESTS}/default"
   fi
 fi
 
+
 echo "Deploying ${TEST_MANIFESTS}..."
 
 kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$?
@@ -215,6 +209,18 @@ if [ "${STORAGE_BACKEND}" == "seaweedfs" ]; then
   echo "SeaweedFS init job completed successfully."
 fi
 
+if [ "${MULTI_USER}" == "true" ]; then
+  echo "Creating KF Profile..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/test-profiles.yaml
+  sleep 30 # Let the profile controler reconcile the namespace
+
+  echo "Applying kubeflow-edit ClusterRole with proper aggregation..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/kubeflow-edit-clusterrole.yaml
+
+  echo "Applying network policy to allow user namespace access to kubeflow services..."
+  kubectl apply -f test_data/kubernetes/seaweedfs/allow-user-namespace-access.yaml
+fi
+
 # Verify pipeline integration for multi-user mode
 if [ "${MULTI_USER}" == "true" ]; then
   echo "Verifying Pipeline Integration..."

.github/workflows/e2e-test.yml

@@ -172,8 +172,15 @@ jobs:
         storage: [ "seaweedfs", "minio"]
         multi_user: [ "true" ]
         test_label: [ "E2ECritical" ]
+        include:
+        - multi_user: "true"
+          artifact_proxy: "true"
+          storage: "seaweedfs"
+          k8s_version: "v1.31.0"
+          cache_enabled: "true"
+          test_label: "E2ECritical" 
       fail-fast: false
-    name: End to End Critical Scenario Multi User Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} multiUser=${{ matrix.multi_user }} argoVersion=${{ matrix.argo_version}} proxy=${{ matrix.proxy}} storage=${{ matrix.storage }}
+    name: End to End Critical Scenario Multi User Tests - K8s ${{ matrix.k8s_version }} cacheEnabled=${{ matrix.cache_enabled }} multiUser=${{ matrix.multi_user }} argoVersion=${{ matrix.argo_version}} proxy=${{ matrix.proxy}} storage=${{ matrix.storage }} artifactProxy=${{ matrix.artifact_proxy }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -196,6 +203,7 @@ jobs:
           cache_enabled: ${{ matrix.cache_enabled }}
           multi_user: "true"
           storage_backend: ${{ matrix.storage }}
+          artifact_proxy: ${{ matrix.artifact_proxy }}
           image_path: ${{ needs.build.outputs.IMAGE_PATH }}
           image_tag: ${{ needs.build.outputs.IMAGE_TAG }}
           image_registry: ${{ needs.build.outputs.IMAGE_REGISTRY }}
@@ -225,6 +233,15 @@ jobs:
           kind --name kfp load docker-image registry.domain.local/modelcar:test
         continue-on-error: true
 
+      - name: Test Artifact Proxy
+        id: test-artifact-proxy
+        if: ${{ steps.deploy.outcome == 'success' && matrix.artifact_proxy == 'true' }}
+        shell: bash
+        env:
+          USER_NAMESPACE: ${{ env.USER_NAMESPACE }}
+        run: |
+          ./test/artifact-proxy/test-artifact-proxy.sh "$USER_NAMESPACE"
+
       - name: Run Tests
         uses: ./.github/actions/test-and-report
         if: ${{ steps.configure.outcome == 'success' }}

manifests/kustomize/base/installs/generic/pipeline-install-config.yaml

@@ -100,5 +100,11 @@ data:
   ## if you specifically need to access unique pipeline root buckets per
   ## namespace that are outside the kubeflow namespace's access.
   ## Valid values are 'true' and 'false'. Defaults to 'false'.
-  ## For more information see: https://github.com/kubeflow/pipelines/issues/11987
+  ## For more information see: https://github.com/kubeflow/pipelines/pull/11965
   ARTIFACTS_PROXY_ENABLED: "false"
+  ## ARTIFACT_RETENTION_DAYS: Defines the retention period (in days) for artifacts stored in SeaweedFS.  
+  ## - A positive integer specifies the number of days after which the artifact will be automatically deleted.  
+  ## - A negative integer (e.g., -1) disables automatic deletion, meaning artifacts are retained indefinitely.  
+  ## Note: The unit of this configuration is always in days.  
+  ARTIFACT_RETENTION_DAYS: "-1"
+

manifests/kustomize/base/installs/multi-user/pipelines-profile-controller/deployment.yaml

@@ -35,7 +35,9 @@ spec:
               name: pipeline-install-config
               key: appVersion
         - name: AWS_ENDPOINT_URL
-          value: http://seaweedfs:8111
+          value: http://seaweedfs.kubeflow:8111
+        - name: S3_ENDPOINT_URL
+          value: http://seaweedfs.kubeflow:8333 
         - name: AWS_REGION
           value: us-east-1
         - name: AWS_ACCESS_KEY_ID
@@ -60,6 +62,11 @@ spec:
               key: ARTIFACTS_PROXY_ENABLED
               name: pipeline-install-config
               optional: true
+        - name: ARTIFACT_RETENTION_DAYS
+          valueFrom:
+            configMapKeyRef:
+              key: ARTIFACT_RETENTION_DAYS
+              name: pipeline-install-config
         volumeMounts:
         - name: hooks
           mountPath: /hooks