From 89cc910a868d6582d94dad6b5e27966f3ed09dbf Mon Sep 17 00:00:00 2001 From: Ege Zambelli Date: Wed, 28 Oct 2026 07:32:16 +0100 Subject: [PATCH] helm: render the filer's mysql credentials once when the user sets them (#12139) * helm: render the filer's mysql credentials once when the user sets them With the MySQL filer store, the filer reads WEED_MYSQL_USERNAME and WEED_MYSQL_PASSWORD from the chart's db secret. Setting either one in extraEnvironmentVars or secretExtraEnvironmentVars, as helm_ci.yml does for the password, added a second entry with the same name: Helm 4 rejects the install (env: duplicate entries for key), and Helm 3 deploys with a warning that one definition may be dropped on apply. The db secret's entry is now left out for a key the user sets, so each key is rendered once and the user's value wins, the rule #8488 applied to a key set in both global and filer extraEnvironmentVars. Without an override the filer StatefulSet is unchanged. * helm: render the selected filer mysql credential ahead of the merged vars A credential the user sets now renders in the early position the db secret's entries used to occupy, and is excluded from the merged and secret loops. Kubernetes resolves a $(VAR) reference only against names declared earlier, so a later entry such as WEED_MYSQL_DSN can again reference the active credential. Secret-backed values win over plain ones; the db secret's entry remains the fallback. * helm: let a secret-backed filer env key replace the plain one secretExtraEnvironmentVars now wins over extraEnvironmentVars for the same name, rendering once at the merged key's sorted position instead of twice. The values.yaml WEED_MYSQL_* defaults no longer duplicate when overridden through the secret map - e.g. enabling the store via a secret WEED_MYSQL_ENABLED also rendered the "false" default before, which Helm 4's server-side apply rejects as a duplicate. * helm: keep a credential that references another env var in sorted position A user-supplied credential whose value itself holds a $(...) reference stays in the merged vars' sorted order so its own references still resolve; only reference-free values, valueFrom entries, and the db-secret defaults move to the early slot. * helm: render vars a filer mysql credential references ahead of it Hoisting the selected WEED_MYSQL_USERNAME/PASSWORD entries above the merged extra vars let WEED_MYSQL_DSN-style $(...) references resolve, but it broke the opposite direction: a user-set credential such as WEED_MYSQL_USERNAME: "$(DB_USER)" was emitted before DB_USER, so Kubernetes left the reference literal. Pull every merged or secret var a credential value transitively references out of the sorted set and emit them first, dependencies before dependents, so both directions resolve. * helm: emit filer mysql credentials after the chart's own env vars The dependency-ordered credential block rendered before SEAWEEDFS_FULLNAME and the s3 credentials, so a user-supplied WEED_MYSQL_USERNAME could not reference them via $(...). Move the block below those fixed entries so references in either direction resolve. * helm: bound filer mysql credential ordering passes by var count Chains deeper than the fixed eight passes forced a credential out ahead of the vars it references. Cap discovery at the number of merged/secret vars and emit passes at the ordered list length, so any resolvable chain finishes before the last-pass fallback emits cyclic leftovers. --------- Co-authored-by: chrislusf --- .github/workflows/helm_ci.yml | 35 ++++++++ .../templates/filer/filer-statefulset.yaml | 89 +++++++++++++++---- 2 files changed, 107 insertions(+), 17 deletions(-) diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index 9d313972e6..cd18fc22c4 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -636,12 +636,47 @@ jobs: --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.name=db \ --set filer.secretExtraEnvironmentVars.WEED_MYSQL_PASSWORD.secretKeyRef.key=password > /tmp/filer-mysql-on-secret.yaml grep -A 4 -- "- name: WEED_MYSQL_PASSWORD$" /tmp/filer-mysql-on-secret.yaml | grep -q "name: db" + # It replaces the db secret's entry rather than adding a second one, which + # Helm 4's server-side apply rejects as a duplicate. + test "$(grep -c -- "- name: WEED_MYSQL_PASSWORD$" /tmp/filer-mysql-on-secret.yaml)" -eq 1 + test "$(grep -c -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-on-secret.yaml)" -eq 1 + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_USERNAME=app > /tmp/filer-mysql-on-plain.yaml + test "$(grep -c -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-on-plain.yaml)" -eq 1 + grep -A 1 -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-on-plain.yaml | grep -q 'value: "app"' + # Credentials render ahead of the merged vars so a $(...) reference resolves. + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_USERNAME=app \ + --set-string 'filer.extraEnvironmentVars.WEED_MYSQL_DSN=$(WEED_MYSQL_USERNAME)@tcp(h)/db' > /tmp/filer-mysql-dsn.yaml + test "$(grep -n -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-dsn.yaml | cut -d: -f1)" -lt "$(grep -n -- "- name: WEED_MYSQL_DSN$" /tmp/filer-mysql-dsn.yaml | cut -d: -f1)" + # Vars a credential's own $(...) references point at render before it. + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set-string filer.extraEnvironmentVars.DB_USER=app \ + --set-string 'filer.extraEnvironmentVars.WEED_MYSQL_USERNAME=$(DB_USER)' > /tmp/filer-mysql-ref.yaml + test "$(grep -n -- "- name: DB_USER$" /tmp/filer-mysql-ref.yaml | cut -d: -f1)" -lt "$(grep -n -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-ref.yaml | cut -d: -f1)" + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set filer.secretExtraEnvironmentVars.DB_USER.secretKeyRef.name=db \ + --set filer.secretExtraEnvironmentVars.DB_USER.secretKeyRef.key=user \ + --set-string 'filer.extraEnvironmentVars.WEED_MYSQL_USERNAME=$(DB_USER)' > /tmp/filer-mysql-ref-secret.yaml + test "$(grep -c -- "- name: DB_USER$" /tmp/filer-mysql-ref-secret.yaml)" -eq 1 + test "$(grep -n -- "- name: DB_USER$" /tmp/filer-mysql-ref-secret.yaml | cut -d: -f1)" -lt "$(grep -n -- "- name: WEED_MYSQL_USERNAME$" /tmp/filer-mysql-ref-secret.yaml | cut -d: -f1)" # A flag the chart cannot read counts as selected, not as off. helm template test $CHART_DIR \ --set filer.secretExtraEnvironmentVars.WEED_MYSQL_ENABLED.secretKeyRef.name=store \ --set filer.secretExtraEnvironmentVars.WEED_MYSQL_ENABLED.secretKeyRef.key=enabled > /tmp/filer-mysql-secret.yaml grep -q "name: test-seaweedfs-db-secret" /tmp/filer-mysql-secret.yaml grep -q "name: WEED_MYSQL_HOSTNAME" /tmp/filer-mysql-secret.yaml + # A secret-backed key also replaces the matching plain entry, not adds one. + test "$(grep -c -- "- name: WEED_MYSQL_ENABLED$" /tmp/filer-mysql-secret.yaml)" -eq 1 + helm template test $CHART_DIR \ + --set-string filer.extraEnvironmentVars.WEED_MYSQL_ENABLED=true \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_HOSTNAME.secretKeyRef.name=db \ + --set filer.secretExtraEnvironmentVars.WEED_MYSQL_HOSTNAME.secretKeyRef.key=host > /tmp/filer-mysql-hostname-secret.yaml + test "$(grep -c -- "- name: WEED_MYSQL_HOSTNAME$" /tmp/filer-mysql-hostname-secret.yaml)" -eq 1 helm template test $CHART_DIR \ --set-string filer.extraEnvironmentVars.WEED_MYSQL2_HOSTNAME=other > /tmp/filer-mysql2.yaml grep -q "name: WEED_MYSQL2_HOSTNAME" /tmp/filer-mysql2.yaml diff --git a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml index 770bfeffec..97276773b5 100644 --- a/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer/filer-statefulset.yaml @@ -101,27 +101,77 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - {{- if $mysqlEnabled }} - - name: WEED_MYSQL_USERNAME - valueFrom: - secretKeyRef: - name: {{ include "seaweedfs.fullname" . }}-db-secret - key: user - optional: true - - name: WEED_MYSQL_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "seaweedfs.fullname" . }}-db-secret - key: password - optional: true - {{- end }} + {{- $mergedExtraEnvironmentVars := dict }} + {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.filer "target" $mergedExtraEnvironmentVars) }} + {{- $secretExtraEnvironmentVars := .Values.filer.secretExtraEnvironmentVars | default dict }} + {{- $emittedEarly := dict }} - name: SEAWEEDFS_FULLNAME value: "{{ include "seaweedfs.fullname" . }}" {{- if and .Values.filer.s3.enabled .Values.filer.s3.enableAuth }} {{- include "seaweedfs.s3.credentialEnv" . | nindent 12 }} {{- end }} - {{- $mergedExtraEnvironmentVars := dict }} - {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.filer "target" $mergedExtraEnvironmentVars) }} + {{- if $mysqlEnabled }} + {{- /* Emit the mysql credentials and every var they reach through $(...) + in dependency order, after the chart's own vars so references to + them resolve too: Kubernetes only resolves a reference to a var + declared earlier. A credential the user sets wins over the db + secret's, secret-backed over plain. */}} + {{- $early := dict "WEED_MYSQL_USERNAME" true "WEED_MYSQL_PASSWORD" true }} + {{- $maxDiscover := int (add (len $mergedExtraEnvironmentVars) (len $secretExtraEnvironmentVars)) }} + {{- range $round := until $maxDiscover }} + {{- range $key := keys $early }} + {{- $v := index $mergedExtraEnvironmentVars $key }} + {{- if and (not (hasKey $secretExtraEnvironmentVars $key)) (kindIs "string" $v) }} + {{- range $ref := regexFindAll `\$\([A-Za-z_][A-Za-z0-9_]*\)` (tpl $v $) -1 }} + {{- $name := $ref | trimPrefix "$(" | trimSuffix ")" }} + {{- if and (or (hasKey $mergedExtraEnvironmentVars $name) (hasKey $secretExtraEnvironmentVars $name)) (not (hasKey $early $name)) }} + {{- $_ := set $early $name true }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- $ordered := list "WEED_MYSQL_USERNAME" "WEED_MYSQL_PASSWORD" }} + {{- range $key := keys $early | sortAlpha }} + {{- if not (has $key $ordered) }}{{ $ordered = append $ordered $key }}{{ end }} + {{- end }} + {{- $maxRounds := int (len $ordered) }} + {{- range $round := until $maxRounds }} + {{- range $key := $ordered }} + {{- if not (hasKey $emittedEarly $key) }} + {{- $ready := true }} + {{- $v := index $mergedExtraEnvironmentVars $key }} + {{- if and (not (hasKey $secretExtraEnvironmentVars $key)) (kindIs "string" $v) }} + {{- range $ref := regexFindAll `\$\([A-Za-z_][A-Za-z0-9_]*\)` (tpl $v $) -1 }} + {{- $name := $ref | trimPrefix "$(" | trimSuffix ")" }} + {{- if and (hasKey $early $name) (not (hasKey $emittedEarly $name)) }}{{- $ready = false }}{{- end }} + {{- end }} + {{- end }} + {{- if or $ready (eq $round (sub $maxRounds 1)) }} + - name: {{ $key }} + {{- if hasKey $secretExtraEnvironmentVars $key }} + valueFrom: {{ toYaml (index $secretExtraEnvironmentVars $key) | nindent 16 }} + {{- else if hasKey $mergedExtraEnvironmentVars $key }} + {{- if kindIs "string" $v }} + value: {{ tpl $v $ | quote }} + {{- else }} + valueFrom: + {{ toYaml $v | nindent 16 | trim }} + {{- end }} + {{- else }} + valueFrom: + secretKeyRef: + name: {{ include "seaweedfs.fullname" $ }}-db-secret + key: {{ ternary "password" "user" (eq $key "WEED_MYSQL_PASSWORD") }} + optional: true + {{- end }} + {{- $_ := set $emittedEarly $key true }} + {{- $_ := unset $mergedExtraEnvironmentVars $key }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} {{- if not $mysqlEnabled }} {{- range $key := keys $mergedExtraEnvironmentVars }} {{- if hasPrefix "WEED_MYSQL_" $key }} @@ -130,6 +180,10 @@ spec: {{- end }} {{- end }} {{- range $key := keys $mergedExtraEnvironmentVars | sortAlpha }} + {{- if hasKey $secretExtraEnvironmentVars $key }} + - name: {{ $key }} + valueFrom: {{ toYaml (index $secretExtraEnvironmentVars $key) | nindent 16 }} + {{- else }} {{- $value := index $mergedExtraEnvironmentVars $key }} - name: {{ $key }} {{- if kindIs "string" $value }} @@ -141,9 +195,10 @@ spec: value: {{ toJson $value | quote }} {{- end -}} {{- end }} + {{- end }} {{- if .Values.filer.secretExtraEnvironmentVars }} {{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }} - {{- if or $mysqlEnabled (not (hasPrefix "WEED_MYSQL_" $key)) }} + {{- if and (or $mysqlEnabled (not (hasPrefix "WEED_MYSQL_" $key))) (not (hasKey $emittedEarly $key)) (not (hasKey $mergedExtraEnvironmentVars $key)) }} - name: {{ $key }} valueFrom: {{ toYaml $value | nindent 16 }} {{- end }}