[Improvement] Separate S3 storage configuration for MLRun and Kubeflow Pipeline

charts/mlrun-ce/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v1
 name: mlrun-ce
-version: 0.11.0-rc.36
+version: 0.11.0-rc.37
 description: MLRun Open Source Stack
 home: https://iguazio.com
 icon: https://www.iguazio.com/wp-content/uploads/2019/10/Iguazio-Logo.png

charts/mlrun-ce/README.md

@@ -317,6 +317,7 @@ $ kubectl --namespace mlrun delete pvc <pv-name>
 $ rm -rf my-mlrun-mlrun-ce-mlrun
 ```
 
+
 ### Using Kubeflow Pipelines
 
 MLRun enables you to run your functions while saving outputs and artifacts in a way that is visible to Kubeflow Pipelines.
@@ -330,4 +331,4 @@ This table shows the versions of the main components in the MLRun CE chart:
 
 | MLRun CE   | MLRun  | Nuclio | Jupyter | MPI Operator | SeaweedFS | Spark Operator | Pipelines | Kube-Prometheus-Stack | OpenTelemetry Operator |
 |------------|--------|--------|---------|--------------|-----------|----------------|-----------|-----------------------|------------------------|
-| **0.11.0** | 1.11.0 | 1.15.9 | 4.5.0   | 0.2.3        | 4.17.0    | 2.1.0          | 2.15.0    | 72.1.1                | 0.78.1                 |
+| **0.11.0-rc.37** | 1.11.0 | 1.15.9 | 4.5.0   | 0.2.3        | 4.17.0    | 2.1.0          | 2.15.0    | 72.1.1                | 0.78.1                 |

charts/mlrun-ce/templates/_helpers.tpl

@@ -151,26 +151,87 @@ S3 Service Port - returns the port for pipeline config
 {{- end -}}
 
 {{/*
-S3 Access Key - uses top-level s3.storage.accessKey for all components (MLRun, Jupyter, Pipelines)
+S3 Access Key - uses top-level storage.s3.accessKey for MLRun and Jupyter
 */}}
 {{- define "mlrun-ce.s3.accessKey" -}}
 {{- .Values.storage.s3.accessKey -}}
 {{- end -}}
 
 {{/*
-S3 Secret Key - uses top-level s3.storage.secretKey for all components (MLRun, Jupyter, Pipelines)
+S3 Secret Key - uses top-level storage.s3.secretKey for MLRun and Jupyter
 */}}
 {{- define "mlrun-ce.s3.secretKey" -}}
 {{- .Values.storage.s3.secretKey -}}
 {{- end -}}
 
 {{/*
-S3 Bucket - uses top-level s3.storage.bucket for all components
+S3 Bucket - uses top-level storage.s3.bucket for MLRun and Jupyter
 */}}
 {{- define "mlrun-ce.s3.bucket" -}}
 {{- .Values.storage.s3.bucket -}}
 {{- end -}}
 
+{{/*
+Pipelines S3 Access Key - falls back to storage.s3.accessKey when not explicitly set.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.accessKey" -}}
+{{- coalesce .Values.pipelines.storage.s3.accessKey .Values.storage.s3.accessKey -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Secret Key - falls back to storage.s3.secretKey when not explicitly set.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.secretKey" -}}
+{{- coalesce .Values.pipelines.storage.s3.secretKey .Values.storage.s3.secretKey -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Bucket - falls back to storage.s3.bucket when not explicitly set.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.bucket" -}}
+{{- coalesce .Values.pipelines.storage.s3.bucket "mlrun" -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Host - SeaweedFS in-cluster for local mode, s3.amazonaws.com for s3 mode.
+Override via pipelines.storage.s3.host for custom endpoints.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.host" -}}
+{{- if .Values.pipelines.storage.s3.host -}}
+{{- .Values.pipelines.storage.s3.host -}}
+{{- else if eq .Values.storage.mode "local" -}}
+{{- include "mlrun-ce.s3.service.host" . -}}
+{{- else -}}
+s3.amazonaws.com
+{{- end -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Port - SeaweedFS port for local mode, 443 for s3 mode.
+Override via pipelines.storage.s3.port for custom endpoints.
+*/}}
+{{- define "mlrun-ce.pipelines.s3.port" -}}
+{{- if .Values.pipelines.storage.s3.port -}}
+{{- .Values.pipelines.storage.s3.port | toString -}}
+{{- else if eq .Values.storage.mode "local" -}}
+{{- include "mlrun-ce.s3.service.port" . -}}
+{{- else -}}
+443
+{{- end -}}
+{{- end -}}
+
+{{/*
+Pipelines S3 Secure / Insecure - local mode uses plain HTTP (insecure=true), all other modes use HTTPS.
+secure returns "true"/"false"; insecure returns the inverse (for workflow-controller artifactRepository).
+*/}}
+{{- define "mlrun-ce.pipelines.s3.secure" -}}
+{{- if eq .Values.storage.mode "local" -}}false{{- else -}}true{{- end -}}
+{{- end -}}
+
+{{- define "mlrun-ce.pipelines.s3.insecure" -}}
+{{- if eq (include "mlrun-ce.pipelines.s3.secure" .) "true" -}}false{{- else -}}true{{- end -}}
+{{- end -}}
+
 {{/*
 Legacy Minio Service URL - kept for backward compatibility
 */}}
@@ -197,7 +258,7 @@ s3://
 {{- end -}}
 
 {{- define "mlrun-ce.artifactPath" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{run.project}}` }}/artifacts
@@ -207,7 +268,7 @@ s3://{{ $bucket }}/projects/{{ `{{run.project}}` }}/artifacts
 {{- end -}}
 
 {{- define "mlrun-ce.featureStore.dataPrefix" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{project}/FeatureStore/{name}/{kind}
@@ -217,7 +278,7 @@ s3://{{ $bucket }}/projects/{project}/FeatureStore/{name}/{kind}
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.userSpace" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
@@ -227,7 +288,7 @@ s3://{{ $bucket }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.application" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
@@ -237,7 +298,7 @@ s3://{{ $bucket }}/users/pipelines/{{ `{{project}}` }}/monitoring-apps/
 {{- end -}}
 
 {{- define "mlrun-ce.model-endpoint.monitoring.default" -}}
-{{- $bucket := .Values.global.infrastructure.aws.bucketName | default "mlrun" -}}
+{{- $bucket := coalesce .Values.global.infrastructure.aws.bucketName .Values.storage.s3.bucket "mlrun" -}}
 {{- $container := .Values.storage.azure.containerName | default "" -}}
 {{- if eq .Values.storage.mode "azure-blob" -}}
 az://{{ $container }}/projects/{{ `{{project}}` }}/model-endpoints/{{ `{{kind}}` }}

charts/mlrun-ce/templates/config/mlrun-env-configmap.yaml

@@ -10,7 +10,7 @@ data:
   MLRUN_HTTPDB__PROJECTS__LEADER: mlrun
   MLRUN_HTTPDB__PROJECTS__FOLLOWERS: nuclio
   # Storage credentials are loaded from the 'storage-credentials' Secret
-  # (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3 for s3, or AZURE_STORAGE_* for azure-blob)
+  # (local: AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY/AWS_ENDPOINT_URL_S3; s3: AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY; azure-blob: AZURE_STORAGE_*)
   MLRUN_FUNCTION__SPEC__SERVICE_ACCOUNT__DEFAULT: {{ .Values.mlrun.api.functionSpecServiceAccountDefault | default "" | quote }}
   MLRUN_HTTPDB__REAL_PATH: {{ include "mlrun-ce.httpdb.realPath" . | quote }}
   MLRUN_ARTIFACT_PATH: {{ include "mlrun-ce.artifactPath" . | quote }}

charts/mlrun-ce/templates/config/storage-secret.yaml

@@ -25,8 +25,7 @@ stringData:
   {{- with .Values.storage.azure.tenantId }}
   AZURE_STORAGE_TENANT_ID: {{ . }}
   {{- end }}
-{{- else }}
-{{- if not .Values.global.infrastructure.aws.s3NonAnonymous }}
+{{- else if not .Values.global.infrastructure.aws.s3NonAnonymous }}
 apiVersion: v1
 kind: Secret
 metadata:
@@ -37,6 +36,7 @@ type: Opaque
 stringData:
   AWS_ACCESS_KEY_ID: {{ include "mlrun-ce.s3.accessKey" . }}
   AWS_SECRET_ACCESS_KEY: {{ include "mlrun-ce.s3.secretKey" . }}
+  {{- if eq .Values.storage.mode "local" }}
   AWS_ENDPOINT_URL_S3: {{ include "mlrun-ce.s3.service.url" . }}
-{{- end }}
+  {{- end }}
 {{- end }}

charts/mlrun-ce/templates/config/storage-validation.yaml

@@ -1,6 +1,9 @@
 {{- if and (eq .Values.storage.mode "s3") (not .Values.storage.s3.bucket) }}
 {{ fail "storage.mode is set to \"s3\" but storage.s3.bucket is not provided. Please set storage.s3.bucket." }}
 {{- end }}
+{{- if and (eq .Values.storage.mode "local") (not .Values.storage.s3.bucket) }}
+{{ fail "storage.mode is set to \"local\" but storage.s3.bucket is not provided. Please set storage.s3.bucket." }}
+{{- end }}
 {{- if and (eq .Values.storage.mode "azure-blob") (not .Values.storage.azure.containerName) }}
 {{ fail "storage.mode is set to \"azure-blob\" but storage.azure.containerName is not provided. Please set storage.azure.containerName." }}
 {{- end }}

charts/mlrun-ce/templates/pipelines/configmaps/pipeline-install-config.yaml

@@ -1,13 +1,13 @@
 {{- if .Values.pipelines.enabled -}}
 apiVersion: v1
 data:
-  objectStoreServiceHost: {{ include "mlrun-ce.s3.service.host" . }}
-  objectStoreServicePort: {{ include "mlrun-ce.s3.service.port" . | quote }}
+  objectStoreServiceHost: {{ include "mlrun-ce.pipelines.s3.host" . }}
+  objectStoreServicePort: {{ include "mlrun-ce.pipelines.s3.port" . | quote }}
   ConMaxLifeTime: 120s
   appName: pipeline
   appVersion: {{ .Values.pipelines.images.apiServer.tag }}
   autoUpdatePipelineDefaultVersion: "false"
-  bucketName: "{{ include "mlrun-ce.s3.bucket" . }}"
+  bucketName: "{{ include "mlrun-ce.pipelines.s3.bucket" . }}"
   cacheDb: cachedb
   cacheImage: {{ .Values.pipelines.images.cacheImage.repository }}:{{ .Values.pipelines.images.cacheImage.tag }}
   cacheNodeRestrictions: "false"

charts/mlrun-ce/templates/pipelines/configmaps/workflow-controller-configmap.yaml

@@ -9,9 +9,9 @@ data:
   artifactRepository: |
     archiveLogs: {{ .Values.pipelines.archiveLogs }}
     s3:
-      endpoint: "{{ include "mlrun-ce.s3.service.host" . }}:{{ include "mlrun-ce.s3.service.port" . }}"
-      bucket: "{{ include "mlrun-ce.s3.bucket" . }}"
-      insecure: true
+      endpoint: "{{ include "mlrun-ce.pipelines.s3.host" . }}:{{ include "mlrun-ce.pipelines.s3.port" . }}"
+      bucket: "{{ include "mlrun-ce.pipelines.s3.bucket" . }}"
+      insecure: {{ include "mlrun-ce.pipelines.s3.insecure" . }}
       accessKeySecret:
         name: mlpipeline-seaweedfs-artifact
         key: accesskey

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline-ui.yaml

@@ -33,9 +33,9 @@ spec:
         - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH
           value: /etc/config/viewer-pod-template.json
         - name: MINIO_HOST
-          value: {{ include "mlrun-ce.s3.service.host" . }}
+          value: {{ include "mlrun-ce.pipelines.s3.host" . }}
         - name: MINIO_PORT
-          value: {{ include "mlrun-ce.s3.service.port" . | quote }}
+          value: {{ include "mlrun-ce.pipelines.s3.port" . | quote }}
         - name: MINIO_NAMESPACE
           # This is required because otherwise the namespace is appended to the MinIO hostname used to fetch artifacts, causing the fetch to fail
           value: ""

charts/mlrun-ce/templates/pipelines/deployments/ml-pipeline.yaml

@@ -75,7 +75,7 @@ spec:
                   name: pipeline-install-config
                   key: objectStoreServicePort
             - name: OBJECTSTORECONFIG_SECURE
-              value: "false"
+              value: {{ include "mlrun-ce.pipelines.s3.secure" . | quote }}
             - name: OBJECTSTORECONFIG_BUCKETNAME
               valueFrom:
                 configMapKeyRef:

charts/mlrun-ce/templates/pipelines/secrets/mlpipeline-seaweedfs-artifact.yaml

@@ -1,8 +1,8 @@
 {{- if .Values.pipelines.enabled -}}
 apiVersion: v1
 data:
-  accesskey: {{ include "mlrun-ce.s3.accessKey" . | b64enc | quote }}
-  secretkey: {{ include "mlrun-ce.s3.secretKey" . | b64enc | quote }}
+  accesskey: {{ include "mlrun-ce.pipelines.s3.accessKey" . | b64enc | quote }}
+  secretkey: {{ include "mlrun-ce.pipelines.s3.secretKey" . | b64enc | quote }}
 kind: Secret
 metadata:
   annotations:

charts/mlrun-ce/values.yaml

@@ -21,17 +21,25 @@ global:
       domainNameCertificate: ~
 
 # =============================================================================
-# S3-compatible storage configuration
-# These credentials are used by MLRun, Jupyter, and Kubeflow Pipelines
-# to access the storage backend.
+# S3-compatible storage configuration for MLRun and Jupyter.
+# Kubeflow Pipelines has its own independent storage block at pipelines.storage.s3.
+# Both default to the bundled SeaweedFS. Change only the block(s) you want to
+# move to an external backend — each component is configured independently.
 # =============================================================================
 
 # storage.mode selects which backend credentials are injected into the 'storage-credentials' Secret.
 # Options:
-#   s3 (default) - uses storage.s3.accessKey/secretKey/bucket with SeaweedFS endpoint
-#   azure-blob   - uses storage.azure.* fields
+#   local (default) - bundled SeaweedFS; injects AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY,
+#                     and AWS_ENDPOINT_URL_S3 pointing at the in-cluster SeaweedFS S3 service.
+#                     Credentials come from storage.s3.accessKey/secretKey; artifact path from
+#                     storage.s3.bucket.
+#   s3              - external AWS S3; injects AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY only
+#                     (no custom endpoint — the SDK uses the standard AWS endpoint).
+#                     Credentials come from storage.s3.accessKey/secretKey; artifact path from
+#                     storage.s3.bucket (or global.infrastructure.aws.bucketName).
+#   azure-blob      - Azure Blob Storage; injects AZURE_STORAGE_* fields from storage.azure.*
 storage:
-  mode: s3
+  mode: local
   s3:
     accessKey: "seaweed"
     secretKey: "seaweed123"
@@ -452,6 +460,18 @@ pipelines:
   name: pipelines
   # Log level for KFP api-server (DEBUG, INFO, WARNING, ERROR)
   logLevel: INFO
+  # S3 storage config for Kubeflow Pipelines artifacts.
+  # accessKey/secretKey/bucket fall back to storage.s3.* when left empty, so by default
+  # pipelines share the same SeaweedFS credentials as MLRun and Jupyter.
+  # host/port default to the in-cluster SeaweedFS endpoint (local mode) or s3.amazonaws.com:443 (s3 mode).
+  # Set any field explicitly to override for a pipelines-specific backend.
+  storage:
+    s3:
+      accessKey: ""
+      secretKey: ""
+      bucket: ""
+      host: ""
+      port: ""
   ui:
     enabled: false
   metadata: