Deploy: added more charts/helm cfgs for OS.

Author: vladd-bit

deploy/Makefile

@@ -11,23 +11,8 @@ REMOTE_SSH_KEY_ARG = $(if $(strip $(REMOTE_SSH_KEY)),-i $(REMOTE_SSH_KEY),)
 HELM_OPENSEARCH_RELEASE ?= cogstack-opensearch
 HELM_OPENSEARCH_NAMESPACE ?= cogstack
 HELM_OPENSEARCH_CHART ?= ./charts/opensearch
-HELM_OPENSEARCH_CONFIG_FILE ?= ../services/elasticsearch/config/opensearch.yml
-HELM_OPENSEARCH_LOG4J_FILE ?= ../services/elasticsearch/config/log4j2_opensearch.properties
-HELM_DASHBOARDS_CONFIG_FILE ?= ../services/kibana/config/opensearch.yml
-HELM_OPENSEARCH_ENV_FILE ?= ./elasticsearch.env
-HELM_OPENSEARCH_USERS_ENV_FILE ?= ../security/env/users_elasticsearch.env
-HELM_OPENSEARCH_CERTS_ENV_FILE ?= ../security/env/certificates_elasticsearch.env
-HELM_OPENSEARCH_SECURITY_DIR ?= ../security/es_roles/opensearch
-HELM_OPENSEARCH_SET_FILES = --set-file configFiles.opensearchRaw=$(HELM_OPENSEARCH_CONFIG_FILE) \
-	--set-file configFiles.log4jRaw=$(HELM_OPENSEARCH_LOG4J_FILE) \
-	--set-file configFiles.dashboardsRaw=$(HELM_DASHBOARDS_CONFIG_FILE) \
-	--set-file envFile.raw=$(HELM_OPENSEARCH_ENV_FILE) \
-	--set-file usersEnvFile.raw=$(HELM_OPENSEARCH_USERS_ENV_FILE) \
-	--set-file certificatesEnvFile.raw=$(HELM_OPENSEARCH_CERTS_ENV_FILE) \
-	--set-file securityFiles.configRaw=$(HELM_OPENSEARCH_SECURITY_DIR)/config.yml \
-	--set-file securityFiles.internalUsersRaw=$(HELM_OPENSEARCH_SECURITY_DIR)/internal_users.yml \
-	--set-file securityFiles.rolesRaw=$(HELM_OPENSEARCH_SECURITY_DIR)/roles.yml \
-	--set-file securityFiles.rolesMappingRaw=$(HELM_OPENSEARCH_SECURITY_DIR)/roles_mapping.yml
+HELM_OPENSEARCH_VALUES_FILE ?= ./helm/opensearch.values.yaml
+HELM_OPENSEARCH_VALUES_ARG = -f $(HELM_OPENSEARCH_VALUES_FILE)
 
 define WITH_ENV
 set -a && source ./export_env_vars.sh;
@@ -89,11 +74,11 @@ load-env: ## Load variables from export_env_vars.sh in a subshell
 show-env: ## Print sorted environment variables after loading export_env_vars.sh
 	${WITH_ENV} >/dev/null 2>&1; printenv | sort
 
-helm-template-opensearch: ## Render OpenSearch chart using shared services/security config files
-	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_SET_FILES)
+helm-template-opensearch: ## Render OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml
+	helm template $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG)
 
-helm-install-opensearch: ## Install/upgrade OpenSearch chart using shared services/security config files
-	helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_SET_FILES) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace
+helm-install-opensearch: ## Install/upgrade OpenSearch chart using chart defaults plus ./helm/opensearch.values.yaml
+	helm upgrade --install $(HELM_OPENSEARCH_RELEASE) $(HELM_OPENSEARCH_CHART) $(HELM_OPENSEARCH_VALUES_ARG) --namespace $(HELM_OPENSEARCH_NAMESPACE) --create-namespace
 
 
 remote-deploy-service: ## Deploy one or more services to a remote machine via SSH + docker compose

deploy/charts/README.md

@@ -16,31 +16,14 @@ This directory contains Helm charts owned by this repository's deployment layer.
 ```bash
 # Render manifests
 helm template cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml
+  -f ./deploy/helm/opensearch.values.yaml
 
 # Install/upgrade
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml \
+  -f ./deploy/helm/opensearch.values.yaml \
   --namespace cogstack --create-namespace
 ```
 
-The OpenSearch and Dashboards config files should come from `services/`, and the security files from `security/`, so Docker and Kubernetes use the same source files.
+The OpenSearch and Dashboards config files should come from `services/`, and the security and env files from `security/` and `deploy/`, so Docker and Kubernetes use the same source files.
+The values file is for cluster-specific overrides only; it does not need to repeat the shared YAML or env file paths.
 Only keys in `envFile.includeKeys`, `usersEnvFile.includeKeys`, and `certificatesEnvFile.includeKeys` are imported.

deploy/charts/opensearch/README.md

@@ -73,16 +73,7 @@ kubectl create secret generic opensearch-certs \
 
 ```bash
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml \
+  -f ./deploy/helm/opensearch.values.yaml \
   --namespace cogstack --create-namespace
 ```
 
@@ -102,27 +93,18 @@ helm upgrade --install cogstack-dashboards ./deploy/charts/opensearch \
 
 ```bash
 helm template cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml
+  -f ./deploy/helm/opensearch.values.yaml
 ```
 
 ## Notes
 
 - Helm templates cannot read arbitrary `../../...` paths directly; `.Files.Get` only sees files packaged inside the chart.
-- In this repo, the chart `files/` entries are symlinked to the shared `services/` and `security/` sources so Docker and Kubernetes stay aligned.
-- The standard install/render commands still use `--set-file` explicitly to make the source-of-truth paths obvious at invocation time.
-- If you run Helm from `deploy/charts/opensearch`, the equivalent relative paths are `../../../services/...` and `../../../security/...`.
-- `envFile.raw` can be set from `deploy/elasticsearch.env`; the chart reads shared values from it (`ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, `KIBANA_SERVER_NAME`) and still generates Kubernetes-specific discovery and publish-host settings itself.
-- `usersEnvFile.raw` can be set from `security/env/users_elasticsearch.env` and feeds only the credential keys required by the enabled components.
-- `certificatesEnvFile.raw` can be set from `security/env/certificates_elasticsearch.env`; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`<name>.pem` / `<name>.key`).
+- In this repo, the chart `files/` entries are symlinked to the shared `deploy/`, `services/`, and `security/` sources so Docker and Kubernetes stay aligned.
+- The standard install/render commands now use `-f ./deploy/helm/opensearch.values.yaml`; that file is for cluster-specific overrides only.
+- The shared `services/`, `security/`, and selected `deploy/` env files are consumed automatically by the chart defaults; you do not need to repeat those paths in the values file.
+- `envFile.raw` defaults to `deploy/elasticsearch.env` and can still be overridden; the chart reads only `ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, and `KIBANA_SERVER_NAME`, while pod IP and discovery hosts remain Kubernetes-specific.
+- `usersEnvFile.raw` defaults to `security/env/users_elasticsearch.env` and can still be overridden; only the credential keys required by the enabled components are imported.
+- `certificatesEnvFile.raw` defaults to `security/env/certificates_elasticsearch.env` and can still be overridden; currently `ES_CLIENT_CERT_NAME` is used to resolve Dashboards cert secret keys (`<name>.pem` / `<name>.key`).
 - `deploy/elasticsearch.env` shared values are used where they make sense on Kubernetes (`ELASTICSEARCH_CLUSTER_NAME`, `ELASTICSEARCH_JAVA_OPTS` / `OPENSEARCH_JAVA_OPTS`, `KIBANA_SERVER_NAME`), while pod IP and discovery hosts remain Kubernetes-specific.
 - By default, `certificates.opensearchNodeFiles[*]` maps pod ordinals `0/1/2` to repo-style node cert keys `elasticsearch-1/2/3`.
 - `opensearch.logPersistence` and `opensearch.performanceAnalyzerPersistence` default to PVC-backed storage to stay closer to the Docker Compose deployment.

deploy/charts/opensearch/files/certificates-elasticsearch.envfile

@@ -0,0 +1 @@
+../../../../security/env/certificates_elasticsearch.env

deploy/charts/opensearch/files/deploy-elasticsearch.envfile

@@ -0,0 +1 @@
+../../../../deploy/elasticsearch.env

deploy/charts/opensearch/files/users-elasticsearch.envfile

@@ -0,0 +1 @@
+../../../../security/env/users_elasticsearch.env

deploy/charts/opensearch/templates/_helpers.tpl

@@ -77,8 +77,9 @@ app.kubernetes.io/component: dashboards
 {{- define "cogstack-opensearch.parsedEnvFile" -}}
 {{- $root := . -}}
 {{- $envData := dict -}}
-{{- if $root.Values.envFile.raw -}}
-{{- $renderedEnv := tpl $root.Values.envFile.raw $root -}}
+{{- $rawEnv := $root.Values.envFile.raw | default ($root.Files.Get "files/deploy-elasticsearch.envfile") -}}
+{{- if $rawEnv -}}
+{{- $renderedEnv := tpl $rawEnv $root -}}
 {{- range $line := splitList "\n" $renderedEnv }}
   {{- $clean := trim (replace "\r" "" $line) -}}
   {{- if and $clean (not (hasPrefix "#" $clean)) -}}
@@ -110,8 +111,9 @@ app.kubernetes.io/component: dashboards
 {{- define "cogstack-opensearch.parsedUsersEnvFile" -}}
 {{- $root := . -}}
 {{- $usersData := dict -}}
-{{- if $root.Values.usersEnvFile.raw -}}
-{{- $renderedUsers := tpl $root.Values.usersEnvFile.raw $root -}}
+{{- $rawUsers := $root.Values.usersEnvFile.raw | default ($root.Files.Get "files/users-elasticsearch.envfile") -}}
+{{- if $rawUsers -}}
+{{- $renderedUsers := tpl $rawUsers $root -}}
 {{- range $line := splitList "\n" $renderedUsers }}
   {{- $clean := trim (replace "\r" "" $line) -}}
   {{- if and $clean (not (hasPrefix "#" $clean)) -}}
@@ -137,8 +139,9 @@ app.kubernetes.io/component: dashboards
 {{- define "cogstack-opensearch.parsedCertificatesEnvFile" -}}
 {{- $root := . -}}
 {{- $certData := dict -}}
-{{- if $root.Values.certificatesEnvFile.raw -}}
-{{- $renderedCerts := tpl $root.Values.certificatesEnvFile.raw $root -}}
+{{- $rawCerts := $root.Values.certificatesEnvFile.raw | default ($root.Files.Get "files/certificates-elasticsearch.envfile") -}}
+{{- if $rawCerts -}}
+{{- $renderedCerts := tpl $rawCerts $root -}}
 {{- range $line := splitList "\n" $renderedCerts }}
   {{- $clean := trim (replace "\r" "" $line) -}}
   {{- if and $clean (not (hasPrefix "#" $clean)) -}}

deploy/charts/opensearch/values.yaml

@@ -12,7 +12,8 @@ configFiles:
   dashboardsRaw: ""
 
 envFile:
-  # Pass deploy/elasticsearch.env via:
+  # Optional override. If empty, chart falls back to files/deploy-elasticsearch.envfile.
+  # Use:
   #   --set-file envFile.raw=./deploy/elasticsearch.env
   raw: ""
   # Only these shared values are read from deploy/elasticsearch.env.
@@ -24,7 +25,8 @@ envFile:
     - KIBANA_SERVER_NAME
 
 usersEnvFile:
-  # Pass security/env/users_elasticsearch.env via:
+  # Optional override. If empty, chart falls back to files/users-elasticsearch.envfile.
+  # Use:
   #   --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env
   raw: ""
   # Only these keys are imported into the credentials Secret.
@@ -34,7 +36,8 @@ usersEnvFile:
     - KIBANA_PASSWORD
 
 certificatesEnvFile:
-  # Pass security/env/certificates_elasticsearch.env via:
+  # Optional override. If empty, chart falls back to files/certificates-elasticsearch.envfile.
+  # Use:
   #   --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env
   raw: ""
   # Only keys listed here are imported from certificates env.

deploy/helm/opensearch.values.yaml

@@ -0,0 +1,29 @@
+# Cluster-specific overrides for the OpenSearch Helm chart.
+#
+# Shared OpenSearch, Dashboards, security YAML, and selected env files are not
+# listed here: the chart already consumes the repo's canonical files from
+# deploy/, services/, and security/ via its bundled defaults.
+
+certificates:
+  opensearchSecretName: opensearch-certs
+  dashboardsSecretName: opensearch-certs
+
+# Uncomment to use a pre-created credentials Secret instead of chart defaults.
+# credentials:
+#   create: false
+#   existingSecret: opensearch-credentials
+
+# Example storage overrides:
+# opensearch:
+#   persistence:
+#     storageClassName: standard
+#   logPersistence:
+#     storageClassName: standard
+#   performanceAnalyzerPersistence:
+#     storageClassName: standard
+#   snapshotBackups:
+#     enabled: true
+#     data:
+#       existingClaim: shared-es-data-backups
+#     config:
+#       existingClaim: shared-es-config-backups

docs/deploy/deployment.md

@@ -70,38 +70,17 @@ Quick usage:
 ```bash
 # render manifests
 helm template cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml
+  -f ./deploy/helm/opensearch.values.yaml
 
 # install or upgrade
 helm upgrade --install cogstack-opensearch ./deploy/charts/opensearch \
-  --set-file configFiles.opensearchRaw=./services/elasticsearch/config/opensearch.yml \
-  --set-file configFiles.log4jRaw=./services/elasticsearch/config/log4j2_opensearch.properties \
-  --set-file configFiles.dashboardsRaw=./services/kibana/config/opensearch.yml \
-  --set-file envFile.raw=./deploy/elasticsearch.env \
-  --set-file usersEnvFile.raw=./security/env/users_elasticsearch.env \
-  --set-file certificatesEnvFile.raw=./security/env/certificates_elasticsearch.env \
-  --set-file securityFiles.configRaw=./security/es_roles/opensearch/config.yml \
-  --set-file securityFiles.internalUsersRaw=./security/es_roles/opensearch/internal_users.yml \
-  --set-file securityFiles.rolesRaw=./security/es_roles/opensearch/roles.yml \
-  --set-file securityFiles.rolesMappingRaw=./security/es_roles/opensearch/roles_mapping.yml \
+  -f ./deploy/helm/opensearch.values.yaml \
   --namespace cogstack --create-namespace
 ```
 
 > The chart expects pre-created Kubernetes Secrets for TLS materials (see the chart README).
-> The `--set-file configFiles.*Raw=...` flags point Helm at the same OpenSearch and Dashboards config files used by Docker Compose.
-> The `--set-file envFile.raw=...` flag lets the chart read shared values from `deploy/elasticsearch.env` while still generating Kubernetes-specific discovery and publish-host settings itself.
-> The `--set-file usersEnvFile.raw=...` flag feeds only the credential keys required by the enabled chart components into the chart Secret.
-> The `--set-file certificatesEnvFile.raw=...` flag loads certificate metadata from `security/env/certificates_elasticsearch.env` (`ES_CLIENT_CERT_NAME` currently).
-> The `--set-file securityFiles.*Raw=...` flags use `security/es_roles/opensearch/*.yml` as the source of OpenSearch security config.
+> The chart already consumes the shared OpenSearch, Dashboards, and security YAML files automatically from this repo.
+> The values file is only for cluster-specific overrides such as secret names, storage classes, replicas, and snapshot PVC claims.
 
 ## 🧰 Makefile Command Overview