diff --git a/backend/src/v2/driver/k8s.go b/backend/src/v2/driver/k8s.go index b4aa89baa85..b69faf60064 100644 --- a/backend/src/v2/driver/k8s.go +++ b/backend/src/v2/driver/k8s.go @@ -17,6 +17,7 @@ package driver import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -178,22 +179,31 @@ func extendPodSpecPatch( // Get node selector information if kubernetesExecutorConfig.GetNodeSelector() != nil { var nodeSelector map[string]string + // skipNodeSelector marks when the node selector input resolved to a null optional + // value. In that case we avoid appending an empty selector to the pod spec. + skipNodeSelector := false if kubernetesExecutorConfig.GetNodeSelector().GetNodeSelectorJson() != nil { err := resolveK8sJsonParameter(ctx, opts, dag, pipeline, mlmd, kubernetesExecutorConfig.GetNodeSelector().GetNodeSelectorJson(), inputParams, &nodeSelector) if err != nil { - return fmt.Errorf("failed to resolve node selector: %w", err) + if errors.Is(err, ErrResolvedParameterNull) { + skipNodeSelector = true + } else { + return fmt.Errorf("failed to resolve node selector: %w", err) + } } } else { nodeSelector = kubernetesExecutorConfig.GetNodeSelector().GetLabels() } - if setOnTaskConfig[pipelinespec.TaskConfigPassthroughType_KUBERNETES_NODE_SELECTOR] { - taskConfig.NodeSelector = nodeSelector - } + if !skipNodeSelector { + if setOnTaskConfig[pipelinespec.TaskConfigPassthroughType_KUBERNETES_NODE_SELECTOR] { + taskConfig.NodeSelector = nodeSelector + } - if setOnPod[pipelinespec.TaskConfigPassthroughType_KUBERNETES_NODE_SELECTOR] { - podSpec.NodeSelector = nodeSelector + if setOnPod[pipelinespec.TaskConfigPassthroughType_KUBERNETES_NODE_SELECTOR] { + podSpec.NodeSelector = nodeSelector + } } } @@ -209,6 +219,9 @@ func extendPodSpecPatch( resolvedParam, err := resolveInputParameter(ctx, dag, pipeline, opts, mlmd, toleration.GetTolerationJson(), inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue // Skip applying the patch for this null/optional parameter + } return fmt.Errorf("failed to resolve toleration: %w", err) } @@ -278,6 +291,9 @@ func extendPodSpecPatch( resolvedSecretName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, secretAsVolume.SecretNameParameter, inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve secret name: %w", err) } secretName = resolvedSecretName.GetStringValue() @@ -337,6 +353,9 @@ func extendPodSpecPatch( resolvedSecretName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, secretAsEnv.SecretNameParameter, inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve secret name: %w", err) } secretName = resolvedSecretName.GetStringValue() @@ -363,12 +382,15 @@ func extendPodSpecPatch( for _, configMapAsVolume := range kubernetesExecutorConfig.GetConfigMapAsVolume() { var configMapName string if configMapAsVolume.ConfigMapNameParameter != nil { - resolvedSecretName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, + resolvedConfigMapName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, configMapAsVolume.ConfigMapNameParameter, inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve configmap name: %w", err) } - configMapName = resolvedSecretName.GetStringValue() + configMapName = resolvedConfigMapName.GetStringValue() } else if configMapAsVolume.ConfigMapName != "" { configMapName = configMapAsVolume.ConfigMapName } else { @@ -424,12 +446,15 @@ func extendPodSpecPatch( var configMapName string if configMapAsEnv.ConfigMapNameParameter != nil { - resolvedSecretName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, + resolvedConfigMapName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, configMapAsEnv.ConfigMapNameParameter, inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve configmap name: %w", err) } - configMapName = resolvedSecretName.GetStringValue() + configMapName = resolvedConfigMapName.GetStringValue() } else if configMapAsEnv.ConfigMapName != "" { configMapName = configMapAsEnv.ConfigMapName } else { @@ -456,6 +481,9 @@ func extendPodSpecPatch( resolvedSecretName, err := resolveInputParameterStr(ctx, dag, pipeline, opts, mlmd, imagePullSecret.SecretNameParameter, inputParams) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve image pull secret name: %w", err) } secretName = resolvedSecretName.GetStringValue() @@ -588,6 +616,9 @@ func extendPodSpecPatch( err := resolveK8sJsonParameter(ctx, opts, dag, pipeline, mlmd, nodeAffinityTerm.GetNodeAffinityJson(), inputParams, &k8sNodeAffinity) if err != nil { + if errors.Is(err, ErrResolvedParameterNull) { + continue + } return fmt.Errorf("failed to resolve node affinity json: %w", err) } diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml new file mode 100644 index 00000000000..de64ad7156e --- /dev/null +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -0,0 +1,657 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Workflow +metadata: + creationTimestamp: null + generateName: missing-kubernetes-optional-inputs-pipeline- +spec: + arguments: + parameters: + - name: kubernetes-comp-log-message-2 + value: '{"nodeAffinity":[{"nodeAffinityJson":{"componentInputParameter":"optional_affinity"}}]}' + - name: components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8 + value: '{"executorLabel":"exec-log-message-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' + - name: implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8 + value: '{"args":["--executor_input","{{$}}","--function_to_execute","log_message"],"command":["sh","-c","\nif + ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 + -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.4'' ''--no-deps'' + ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 + \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" + \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 + -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport + kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef + log_message(message: str) -\u003e None:\n print(message)\n\n"],"image":"python:3.9"}' + - name: components-comp-condition-1 + value: '{"dag":{"tasks":{"log-message-2":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node + affinity applied"}}}},"taskInfo":{"name":"log-message-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_affinity":{"parameterType":"BOOLEAN"}}}}' + - name: kubernetes-comp-log-message-3 + value: '{"nodeSelector":{"nodeSelectorJson":{"componentInputParameter":"optional_selector"}}}' + - name: components-comp-condition-2 + value: '{"dag":{"tasks":{"log-message-3":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-3"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node + selector applied"}}}},"taskInfo":{"name":"log-message-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_selector":{"parameterType":"BOOLEAN"}}}}' + - name: kubernetes-comp-log-message-4 + value: '{"secretAsEnv":[{"keyToEnv":[{"envVar":"PASSWORD","secretKey":"password"}],"secretNameParameter":{"componentInputParameter":"optional_secret_name"}}]}' + - name: components-comp-condition-3 + value: '{"dag":{"tasks":{"log-message-4":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-4"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"secret + env applied"}}}},"taskInfo":{"name":"log-message-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_secret_env":{"parameterType":"BOOLEAN"}}}}' + - name: kubernetes-comp-log-message-5 + value: '{"configMapAsEnv":[{"configMapNameParameter":{"componentInputParameter":"optional_config_map_name"},"keyToEnv":[{"configMapKey":"setting","envVar":"SETTING"}]}]}' + - name: components-comp-condition-4 + value: '{"dag":{"tasks":{"log-message-5":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"configmap + env applied"}}}},"taskInfo":{"name":"log-message-5"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_config_map_env":{"parameterType":"BOOLEAN"}}}}' + - name: kubernetes-comp-log-message-6 + value: '{"tolerations":[{"tolerationJson":{"componentInputParameter":"optional_tolerations"}}]}' + - name: components-comp-condition-5 + value: '{"dag":{"tasks":{"log-message-6":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-6"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"tolerations + applied"}}}},"taskInfo":{"name":"log-message-6"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_tolerations":{"parameterType":"BOOLEAN"}}}}' + - name: components-root + value: '{"dag":{"tasks":{"condition-1":{"componentRef":{"name":"comp-condition-1"},"inputs":{"parameters":{"pipelinechannel--enable_affinity":{"componentInputParameter":"enable_affinity"}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_affinity''] + == true"}},"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--enable_selector":{"componentInputParameter":"enable_selector"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_selector''] + == true"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--enable_secret_env":{"componentInputParameter":"enable_secret_env"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_secret_env''] + == true"}},"condition-4":{"componentRef":{"name":"comp-condition-4"},"inputs":{"parameters":{"pipelinechannel--enable_config_map_env":{"componentInputParameter":"enable_config_map_env"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_config_map_env''] + == true"}},"condition-5":{"componentRef":{"name":"comp-condition-5"},"inputs":{"parameters":{"pipelinechannel--enable_tolerations":{"componentInputParameter":"enable_tolerations"}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_tolerations''] + == true"}},"log-message":{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline + task"}}}},"taskInfo":{"name":"log-message"}}}},"inputDefinitions":{"parameters":{"enable_affinity":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_config_map_env":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_secret_env":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_selector":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_tolerations":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"optional_affinity":{"isOptional":true,"parameterType":"STRUCT"},"optional_config_map_name":{"isOptional":true,"parameterType":"STRING"},"optional_secret_name":{"isOptional":true,"parameterType":"STRING"},"optional_selector":{"isOptional":true,"parameterType":"STRUCT"},"optional_tolerations":{"isOptional":true,"parameterType":"LIST"}}}}' + entrypoint: entrypoint + podMetadata: + annotations: + pipelines.kubeflow.org/v2_component: "true" + labels: + pipelines.kubeflow.org/v2_component: "true" + serviceAccountName: pipeline-runner + templates: + - container: + args: + - --type + - CONTAINER + - --pipeline_name + - missing-kubernetes-optional-inputs-pipeline + - --run_id + - '{{workflow.uid}}' + - --run_name + - '{{workflow.name}}' + - --run_display_name + - "" + - --dag_execution_id + - '{{inputs.parameters.parent-dag-id}}' + - --component + - '{{inputs.parameters.component}}' + - --task + - '{{inputs.parameters.task}}' + - --task_name + - '{{inputs.parameters.task-name}}' + - --container + - '{{inputs.parameters.container}}' + - --iteration_index + - '{{inputs.parameters.iteration-index}}' + - --cached_decision_path + - '{{outputs.parameters.cached-decision.path}}' + - --pod_spec_patch_path + - '{{outputs.parameters.pod-spec-patch.path}}' + - --condition_path + - '{{outputs.parameters.condition.path}}' + - --kubernetes_config + - '{{inputs.parameters.kubernetes-config}}' + - --http_proxy + - "" + - --https_proxy + - "" + - --no_proxy + - "" + command: + - driver + image: ghcr.io/kubeflow/kfp-driver:latest + name: "" + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi + inputs: + parameters: + - name: component + - name: task + - name: container + - name: task-name + - name: parent-dag-id + - default: "-1" + name: iteration-index + - default: "" + name: kubernetes-config + metadata: {} + name: system-container-driver + outputs: + parameters: + - name: pod-spec-patch + valueFrom: + default: "" + path: /tmp/outputs/pod-spec-patch + - default: "false" + name: cached-decision + valueFrom: + default: "false" + path: /tmp/outputs/cached-decision + - name: condition + valueFrom: + default: "true" + path: /tmp/outputs/condition + - dag: + tasks: + - arguments: + parameters: + - name: pod-spec-patch + value: '{{inputs.parameters.pod-spec-patch}}' + name: executor + template: system-container-impl + when: '{{inputs.parameters.cached-decision}} != true' + inputs: + parameters: + - name: pod-spec-patch + - default: "false" + name: cached-decision + metadata: {} + name: system-container-executor + outputs: {} + - container: + command: + - should-be-overridden-during-runtime + env: + - name: KFP_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KFP_POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + envFrom: + - configMapRef: + name: metadata-grpc-configmap + optional: true + image: gcr.io/ml-pipeline/should-be-overridden-during-runtime + name: "" + resources: {} + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + - mountPath: /gcs + name: gcs-scratch + - mountPath: /s3 + name: s3-scratch + - mountPath: /minio + name: minio-scratch + - mountPath: /.local + name: dot-local-scratch + - mountPath: /.cache + name: dot-cache-scratch + - mountPath: /.config + name: dot-config-scratch + initContainers: + - args: + - --copy + - /kfp-launcher/launch + command: + - launcher-v2 + image: ghcr.io/kubeflow/kfp-launcher:latest + name: kfp-launcher + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 100m + volumeMounts: + - mountPath: /kfp-launcher + name: kfp-launcher + inputs: + parameters: + - name: pod-spec-patch + metadata: {} + name: system-container-impl + outputs: {} + podSpecPatch: '{{inputs.parameters.pod-spec-patch}}' + volumes: + - emptyDir: {} + name: kfp-launcher + - emptyDir: {} + name: gcs-scratch + - emptyDir: {} + name: s3-scratch + - emptyDir: {} + name: minio-scratch + - emptyDir: {} + name: dot-local-scratch + - emptyDir: {} + name: dot-cache-scratch + - emptyDir: {} + name: dot-config-scratch + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node + affinity applied"}}}},"taskInfo":{"name":"log-message-2"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message-2 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-log-message-2}}' + name: log-message-2-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-2-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-2-driver.outputs.parameters.cached-decision}}' + depends: log-message-2-driver.Succeeded + name: log-message-2 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-1 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-3"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node + selector applied"}}}},"taskInfo":{"name":"log-message-3"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message-3 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-log-message-3}}' + name: log-message-3-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-3-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-3-driver.outputs.parameters.cached-decision}}' + depends: log-message-3-driver.Succeeded + name: log-message-3 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-2 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-4"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"secret + env applied"}}}},"taskInfo":{"name":"log-message-4"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message-4 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-log-message-4}}' + name: log-message-4-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-4-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-4-driver.outputs.parameters.cached-decision}}' + depends: log-message-4-driver.Succeeded + name: log-message-4 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-3 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"configmap + env applied"}}}},"taskInfo":{"name":"log-message-5"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message-5 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-log-message-5}}' + name: log-message-5-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-5-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-5-driver.outputs.parameters.cached-decision}}' + depends: log-message-5-driver.Succeeded + name: log-message-5 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-4 + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-6"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"tolerations + applied"}}}},"taskInfo":{"name":"log-message-6"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message-6 + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: kubernetes-config + value: '{{workflow.parameters.kubernetes-comp-log-message-6}}' + name: log-message-6-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-6-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-6-driver.outputs.parameters.cached-decision}}' + depends: log-message-6-driver.Succeeded + name: log-message-6 + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: comp-condition-5 + outputs: {} + - container: + args: + - --type + - '{{inputs.parameters.driver-type}}' + - --pipeline_name + - missing-kubernetes-optional-inputs-pipeline + - --run_id + - '{{workflow.uid}}' + - --run_name + - '{{workflow.name}}' + - --run_display_name + - "" + - --dag_execution_id + - '{{inputs.parameters.parent-dag-id}}' + - --component + - '{{inputs.parameters.component}}' + - --task + - '{{inputs.parameters.task}}' + - --task_name + - '{{inputs.parameters.task-name}}' + - --runtime_config + - '{{inputs.parameters.runtime-config}}' + - --iteration_index + - '{{inputs.parameters.iteration-index}}' + - --execution_id_path + - '{{outputs.parameters.execution-id.path}}' + - --iteration_count_path + - '{{outputs.parameters.iteration-count.path}}' + - --condition_path + - '{{outputs.parameters.condition.path}}' + - --http_proxy + - "" + - --https_proxy + - "" + - --no_proxy + - "" + command: + - driver + image: ghcr.io/kubeflow/kfp-driver:latest + name: "" + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 64Mi + inputs: + parameters: + - name: component + - default: "" + name: runtime-config + - default: "" + name: task + - default: "" + name: task-name + - default: "0" + name: parent-dag-id + - default: "-1" + name: iteration-index + - default: DAG + name: driver-type + metadata: {} + name: system-dag-driver + outputs: + parameters: + - name: execution-id + valueFrom: + path: /tmp/outputs/execution-id + - name: iteration-count + valueFrom: + default: "0" + path: /tmp/outputs/iteration-count + - name: condition + valueFrom: + default: "true" + path: /tmp/outputs/condition + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-1}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-1"},"inputs":{"parameters":{"pipelinechannel--enable_affinity":{"componentInputParameter":"enable_affinity"}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_affinity''] + == true"}}' + - name: task-name + value: condition-1 + name: condition-1-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' + depends: condition-1-driver.Succeeded + name: condition-1 + template: comp-condition-1 + when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-2}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--enable_selector":{"componentInputParameter":"enable_selector"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_selector''] + == true"}}' + - name: task-name + value: condition-2 + name: condition-2-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' + depends: condition-2-driver.Succeeded + name: condition-2 + template: comp-condition-2 + when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-3}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--enable_secret_env":{"componentInputParameter":"enable_secret_env"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_secret_env''] + == true"}}' + - name: task-name + value: condition-3 + name: condition-3-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' + depends: condition-3-driver.Succeeded + name: condition-3 + template: comp-condition-3 + when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-4}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-4"},"inputs":{"parameters":{"pipelinechannel--enable_config_map_env":{"componentInputParameter":"enable_config_map_env"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_config_map_env''] + == true"}}' + - name: task-name + value: condition-4 + name: condition-4-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' + depends: condition-4-driver.Succeeded + name: condition-4 + template: comp-condition-4 + when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-comp-condition-5}}' + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + - name: task + value: '{"componentRef":{"name":"comp-condition-5"},"inputs":{"parameters":{"pipelinechannel--enable_tolerations":{"componentInputParameter":"enable_tolerations"}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_tolerations''] + == true"}}' + - name: task-name + value: condition-5 + name: condition-5-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' + - name: condition + value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' + depends: condition-5-driver.Succeeded + name: condition-5 + template: comp-condition-5 + when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline + task"}}}},"taskInfo":{"name":"log-message"}}' + - name: container + value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + - name: task-name + value: log-message + - name: parent-dag-id + value: '{{inputs.parameters.parent-dag-id}}' + name: log-message-driver + template: system-container-driver + - arguments: + parameters: + - name: pod-spec-patch + value: '{{tasks.log-message-driver.outputs.parameters.pod-spec-patch}}' + - default: "false" + name: cached-decision + value: '{{tasks.log-message-driver.outputs.parameters.cached-decision}}' + depends: log-message-driver.Succeeded + name: log-message + template: system-container-executor + inputs: + parameters: + - name: parent-dag-id + metadata: {} + name: root + outputs: {} + - dag: + tasks: + - arguments: + parameters: + - name: component + value: '{{workflow.parameters.components-root}}' + - name: runtime-config + value: '{"parameterValues":{"enable_affinity":false,"enable_config_map_env":false,"enable_secret_env":false,"enable_selector":false,"enable_tolerations":false,"optional_affinity":null,"optional_config_map_name":null,"optional_secret_name":null,"optional_selector":null,"optional_tolerations":null}}' + - name: driver-type + value: ROOT_DAG + name: root-driver + template: system-dag-driver + - arguments: + parameters: + - name: parent-dag-id + value: '{{tasks.root-driver.outputs.parameters.execution-id}}' + - name: condition + value: "" + depends: root-driver.Succeeded + name: root + template: root + inputs: {} + metadata: {} + name: entrypoint + outputs: {} +status: + finishedAt: null + startedAt: null diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 0da72091594..fbab043791b 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.9","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-limit","accelerator-type","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","accelerator-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -351,15 +351,15 @@ spec: - name: component value: '{{workflow.parameters.components-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-limit","accelerator-type","cpu-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-type","cpu-limit","accelerator-limit","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-0886dd8babaca613b2fca24c656568012544ec9eccff86938a7b3859deae5309}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: memory-limit.Succeeded && accelerator-limit.Succeeded && accelerator-type.Succeeded - && cpu-limit.Succeeded + depends: accelerator-type.Succeeded && cpu-limit.Succeeded && accelerator-limit.Succeeded + && memory-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py new file mode 100644 index 00000000000..82c2653fbcf --- /dev/null +++ b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py @@ -0,0 +1,76 @@ +from typing import Any, Dict, List, Optional + +from kfp import dsl, kubernetes + + +@dsl.component +def log_message(message: str) -> None: + print(message) + + +@dsl.pipeline +def missing_kubernetes_optional_inputs_pipeline( + optional_affinity: Optional[Dict[str, Any]] = None, + enable_affinity: bool = False, + optional_selector: Optional[Dict[str, Any]] = None, + enable_selector: bool = False, + optional_secret_name: Optional[str] = None, + enable_secret_env: bool = False, + optional_config_map_name: Optional[str] = None, + enable_config_map_env: bool = False, + optional_tolerations: Optional[List[Dict[str, Any]]] = None, + enable_tolerations: bool = False, +): + base_task = log_message(message="baseline task") + base_task.set_caching_options(enable_caching=False) + + with dsl.If(enable_affinity == True): + affinity_task = log_message(message="node affinity applied") + affinity_task.set_caching_options(enable_caching=False) + kubernetes.add_node_affinity_json( + task=affinity_task, + node_affinity_json=optional_affinity, + ) + + with dsl.If(enable_selector == True): + selector_task = log_message(message="node selector applied") + selector_task.set_caching_options(enable_caching=False) + kubernetes.add_node_selector_json( + task=selector_task, + node_selector_json=optional_selector, + ) + + with dsl.If(enable_secret_env == True): + secret_task = log_message(message="secret env applied") + secret_task.set_caching_options(enable_caching=False) + kubernetes.use_secret_as_env( + task=secret_task, + secret_name=optional_secret_name, + secret_key_to_env={"password": "PASSWORD"}, + ) + + with dsl.If(enable_config_map_env == True): + config_map_task = log_message(message="configmap env applied") + config_map_task.set_caching_options(enable_caching=False) + kubernetes.use_config_map_as_env( + task=config_map_task, + config_map_name=optional_config_map_name, + config_map_key_to_env={"setting": "SETTING"}, + ) + + with dsl.If(enable_tolerations == True): + toleration_task = log_message(message="tolerations applied") + toleration_task.set_caching_options(enable_caching=False) + kubernetes.add_toleration_json( + task=toleration_task, + toleration_json=optional_tolerations, + ) + + +if __name__ == "__main__": + from kfp import compiler + + compiler.Compiler().compile( + pipeline_func=missing_kubernetes_optional_inputs_pipeline, + package_path="missing_kubernetes_optional_inputs.yaml", + ) diff --git a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml new file mode 100644 index 00000000000..30a61bf9a31 --- /dev/null +++ b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml @@ -0,0 +1,456 @@ +# PIPELINE DEFINITION +# Name: missing-kubernetes-optional-inputs-pipeline +# Inputs: +# enable_affinity: bool [Default: False] +# enable_config_map_env: bool [Default: False] +# enable_secret_env: bool [Default: False] +# enable_selector: bool [Default: False] +# enable_tolerations: bool [Default: False] +# optional_affinity: dict +# optional_config_map_name: str +# optional_secret_name: str +# optional_selector: dict +# optional_tolerations: list +components: + comp-condition-1: + dag: + tasks: + log-message-2: + cachingOptions: {} + componentRef: + name: comp-log-message-2 + inputs: + parameters: + message: + runtimeValue: + constant: node affinity applied + taskInfo: + name: log-message-2 + inputDefinitions: + parameters: + pipelinechannel--enable_affinity: + parameterType: BOOLEAN + comp-condition-2: + dag: + tasks: + log-message-3: + cachingOptions: {} + componentRef: + name: comp-log-message-3 + inputs: + parameters: + message: + runtimeValue: + constant: node selector applied + taskInfo: + name: log-message-3 + inputDefinitions: + parameters: + pipelinechannel--enable_selector: + parameterType: BOOLEAN + comp-condition-3: + dag: + tasks: + log-message-4: + cachingOptions: {} + componentRef: + name: comp-log-message-4 + inputs: + parameters: + message: + runtimeValue: + constant: secret env applied + taskInfo: + name: log-message-4 + inputDefinitions: + parameters: + pipelinechannel--enable_secret_env: + parameterType: BOOLEAN + comp-condition-4: + dag: + tasks: + log-message-5: + cachingOptions: {} + componentRef: + name: comp-log-message-5 + inputs: + parameters: + message: + runtimeValue: + constant: configmap env applied + taskInfo: + name: log-message-5 + inputDefinitions: + parameters: + pipelinechannel--enable_config_map_env: + parameterType: BOOLEAN + comp-condition-5: + dag: + tasks: + log-message-6: + cachingOptions: {} + componentRef: + name: comp-log-message-6 + inputs: + parameters: + message: + runtimeValue: + constant: tolerations applied + taskInfo: + name: log-message-6 + inputDefinitions: + parameters: + pipelinechannel--enable_tolerations: + parameterType: BOOLEAN + comp-log-message: + executorLabel: exec-log-message + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-log-message-2: + executorLabel: exec-log-message-2 + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-log-message-3: + executorLabel: exec-log-message-3 + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-log-message-4: + executorLabel: exec-log-message-4 + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-log-message-5: + executorLabel: exec-log-message-5 + inputDefinitions: + parameters: + message: + parameterType: STRING + comp-log-message-6: + executorLabel: exec-log-message-6 + inputDefinitions: + parameters: + message: + parameterType: STRING +deploymentSpec: + executors: + exec-log-message: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 + exec-log-message-2: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 + exec-log-message-3: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 + exec-log-message-4: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 + exec-log-message-5: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 + exec-log-message-6: + container: + args: + - --executor_input + - '{{$}}' + - --function_to_execute + - log_message + command: + - sh + - -c + - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ + \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ + $0\" \"$@\"\n" + - sh + - -ec + - 'program_path=$(mktemp -d) + + + printf "%s" "$0" > "$program_path/ephemeral_component.py" + + _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" + + ' + - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ + \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" + image: python:3.9 +pipelineInfo: + name: missing-kubernetes-optional-inputs-pipeline +root: + dag: + tasks: + condition-1: + componentRef: + name: comp-condition-1 + inputs: + parameters: + pipelinechannel--enable_affinity: + componentInputParameter: enable_affinity + taskInfo: + name: condition-1 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--enable_affinity'] == + true + condition-2: + componentRef: + name: comp-condition-2 + inputs: + parameters: + pipelinechannel--enable_selector: + componentInputParameter: enable_selector + taskInfo: + name: condition-2 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--enable_selector'] == + true + condition-3: + componentRef: + name: comp-condition-3 + inputs: + parameters: + pipelinechannel--enable_secret_env: + componentInputParameter: enable_secret_env + taskInfo: + name: condition-3 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--enable_secret_env'] + == true + condition-4: + componentRef: + name: comp-condition-4 + inputs: + parameters: + pipelinechannel--enable_config_map_env: + componentInputParameter: enable_config_map_env + taskInfo: + name: condition-4 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--enable_config_map_env'] + == true + condition-5: + componentRef: + name: comp-condition-5 + inputs: + parameters: + pipelinechannel--enable_tolerations: + componentInputParameter: enable_tolerations + taskInfo: + name: condition-5 + triggerPolicy: + condition: inputs.parameter_values['pipelinechannel--enable_tolerations'] + == true + log-message: + cachingOptions: {} + componentRef: + name: comp-log-message + inputs: + parameters: + message: + runtimeValue: + constant: baseline task + taskInfo: + name: log-message + inputDefinitions: + parameters: + enable_affinity: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + enable_config_map_env: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + enable_secret_env: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + enable_selector: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + enable_tolerations: + defaultValue: false + isOptional: true + parameterType: BOOLEAN + optional_affinity: + isOptional: true + parameterType: STRUCT + optional_config_map_name: + isOptional: true + parameterType: STRING + optional_secret_name: + isOptional: true + parameterType: STRING + optional_selector: + isOptional: true + parameterType: STRUCT + optional_tolerations: + isOptional: true + parameterType: LIST +schemaVersion: 2.1.0 +sdkVersion: kfp-2.14.4 +--- +platforms: + kubernetes: + deploymentSpec: + executors: + exec-log-message-2: + nodeAffinity: + - nodeAffinityJson: + componentInputParameter: optional_affinity + exec-log-message-3: + nodeSelector: + nodeSelectorJson: + componentInputParameter: optional_selector + exec-log-message-4: + secretAsEnv: + - keyToEnv: + - envVar: PASSWORD + secretKey: password + secretNameParameter: + componentInputParameter: optional_secret_name + exec-log-message-5: + configMapAsEnv: + - configMapNameParameter: + componentInputParameter: optional_config_map_name + keyToEnv: + - configMapKey: setting + envVar: SETTING + exec-log-message-6: + tolerations: + - tolerationJson: + componentInputParameter: optional_tolerations