mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
helm: gate the resize hook policy on the same lookup as its Job
The policy rendered whenever the hook was enabled, which is the default, so every release carrying networkPolicy got one - as a pre-install hook that Helm never collects, left in the namespace after uninstall. It also made egress.kubeApiServer.cidrs mandatory for every release, since the policy claims the API server, for a Job that only runs on an upgrade that grows a PVC. Move the command computation the Job is gated on into a helper and read it from both.
This commit is contained in:
Chris Lu
@@ -921,10 +921,8 @@ jobs:
|
||||
if comp not in pols:
|
||||
failed.append(f"{comp}: workload has no NetworkPolicy")
|
||||
for comp in pols:
|
||||
# The hook Jobs are covered too, and the resize hook Job only renders
|
||||
# when a cluster lookup says a PVC needs growing, so it is never in the
|
||||
# rendered set here.
|
||||
if comp not in wls and comp not in ("bucket-hook", "volume-resize-hook"):
|
||||
# The bucket hook Job is covered too, and a Job is not a workload here.
|
||||
if comp not in wls and comp != "bucket-hook":
|
||||
failed.append(f"{comp}: NetworkPolicy selects a component that is not deployed")
|
||||
|
||||
# The ports a component listens on come from the same values as its
|
||||
@@ -1077,22 +1075,23 @@ jobs:
|
||||
# The API server rule goes only to the components that talk to it.
|
||||
apiserver = {c for c, p in pols.items()
|
||||
if any("ipBlock" in t for r in p["spec"]["egress"] for t in r.get("to") or [])}
|
||||
expected = {"admin", "objectstorage-provisioner", "volume-resize-hook"}
|
||||
expected = {"admin", "objectstorage-provisioner"}
|
||||
if apiserver != expected:
|
||||
failed.append(f"API server egress granted to {sorted(apiserver)}, expected {sorted(expected)}")
|
||||
else:
|
||||
print(f"API server egress limited to {sorted(expected)}")
|
||||
|
||||
# The resize hook runs as a pre-install hook at weight 0, before the release
|
||||
# manifest is applied, so its policy has to be a hook itself and has to sort
|
||||
# ahead of the Job.
|
||||
rh = pols["volume-resize-hook"]["metadata"].get("annotations", {})
|
||||
if rh.get("helm.sh/hook") != "pre-install,pre-upgrade":
|
||||
failed.append(f"volume-resize-hook policy is not a pre-install hook: {rh}")
|
||||
elif int(rh.get("helm.sh/hook-weight", 0)) >= 0:
|
||||
failed.append(f"volume-resize-hook policy weight {rh.get('helm.sh/hook-weight')} does not sort before the Job at 0")
|
||||
# The resize hook's policy is gated on the same lookup as its Job, so
|
||||
# neither is ever in a rendered set - a policy on its own would be an
|
||||
# orphaned hook resource on every install, since Helm does not collect
|
||||
# those. The hook annotations it carries when the lookup does hit are
|
||||
# only reachable against a live cluster.
|
||||
if "volume-resize-hook" in pols:
|
||||
failed.append("volume-resize-hook policy rendered without its Job; the two "
|
||||
"are gated on seaweedfs.volumeResizeHookCommands and have to "
|
||||
"appear together")
|
||||
else:
|
||||
print("volume-resize-hook policy is a pre-install hook ahead of the Job")
|
||||
print("volume-resize-hook policy tracks its Job rather than rendering always")
|
||||
|
||||
# The bucket hook is post-install, so the release manifest is already applied;
|
||||
# its policy must be a plain resource that uninstall cleans up.
|
||||
|
||||
@@ -360,6 +360,65 @@ true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* The kubectl commands the volume resize hook has to run, one per line: a
|
||||
cascade-orphan delete for every StatefulSet whose volumeClaimTemplates no
|
||||
longer match the values, and a patch for every PVC the values grew.
|
||||
|
||||
Empty when there is nothing to resize, which is what gates the Job. Read by
|
||||
its NetworkPolicy too, which has to appear exactly when the Job does - a Job
|
||||
without its policy hangs in a default-deny namespace, and a policy without
|
||||
its Job is an orphaned hook resource on every install.
|
||||
|
||||
Built on lookup, so it is always empty under helm template and on a fresh
|
||||
install, where there is no StatefulSet to compare against yet. */}}
|
||||
{{- define "seaweedfs.volumeResizeHookCommands" -}}
|
||||
{{- $seaweedfsName := include "seaweedfs.fullname" $ }}
|
||||
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- $commands := list }}
|
||||
{{- if .Values.volume.resizeHook.enabled }}
|
||||
{{- range $vname, $volume := $volumes }}
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
|
||||
{{- if $volume.enabled }}
|
||||
{{- $replicas := int $volume.replicas }}
|
||||
{{- $statefulsetName := printf "%s-%s" $seaweedfsName $volumeName }}
|
||||
{{- $statefulset := (lookup "apps/v1" "StatefulSet" $.Release.Namespace $statefulsetName) }}
|
||||
{{- /* Check for changes in volumeClaimTemplates */}}
|
||||
{{- if $statefulset }}
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
{{- if eq .type "persistentVolumeClaim" }}
|
||||
{{- $desiredSize := .size }}
|
||||
{{- range $statefulset.spec.volumeClaimTemplates }}
|
||||
{{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) }}
|
||||
{{- $commands = append $commands (printf "kubectl delete statefulset %s --cascade=orphan" $statefulsetName) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- /* Check for the need for patching existing PVCs */}}
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
{{- if eq .type "persistentVolumeClaim" }}
|
||||
{{- $desiredSize := .size }}
|
||||
{{- range $i, $e := until $replicas }}
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- join "\n" $commands }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}}
|
||||
{{- define "seaweedfs.s3.tlsArgs" -}}
|
||||
{{- $prefix := .prefix -}}
|
||||
|
||||
@@ -39,7 +39,6 @@
|
||||
{{- $targets = append $targets (dict "component" "master" "ports" $ports) }}
|
||||
{{- end }}
|
||||
{{- /* Every volume group carries its own component label and its own ports. */}}
|
||||
{{- $anyVolume := false }}
|
||||
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- range $vname, $volume := $volumes }}
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
@@ -50,7 +49,6 @@
|
||||
{{- $ports = append $ports $volume.metricsPort }}
|
||||
{{- end }}
|
||||
{{- $targets = append $targets (dict "component" $volumeName "ports" $ports) }}
|
||||
{{- $anyVolume = true }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.enabled }}
|
||||
@@ -115,11 +113,13 @@
|
||||
{{- if include "seaweedfs.bucketHookEnabled" . }}
|
||||
{{- $targets = append $targets (dict "component" "bucket-hook" "ports" (list)) }}
|
||||
{{- end }}
|
||||
{{- /* Whether the resize hook Job materialises depends on a cluster lookup, so
|
||||
there is nothing to key on at render time. Its policy is created whenever
|
||||
the hook is enabled and selects no pods on the upgrades that need no
|
||||
resize. */}}
|
||||
{{- if and .Values.volume.resizeHook.enabled $anyVolume }}
|
||||
{{- /* Same cluster lookup the Job is gated on, so the policy appears on exactly
|
||||
the upgrades that grow a PVC and never on the ones that do not. Helm does
|
||||
not garbage-collect hook resources, so a policy rendered unconditionally
|
||||
would be left in the namespace by every install that uninstalls again -
|
||||
and would make egress.kubeApiServer.cidrs mandatory for every release,
|
||||
for a Job most of them never run. */}}
|
||||
{{- if include "seaweedfs.volumeResizeHookCommands" . | trim }}
|
||||
{{- $targets = append $targets (dict "component" "volume-resize-hook" "ports" (list) "apiserver" true "hook" "pre-install,pre-upgrade" "hookWeight" "-10") }}
|
||||
{{- end }}
|
||||
|
||||
|
||||
@@ -1,54 +1,10 @@
|
||||
{{- $seaweedfsName := include "seaweedfs.fullname" $ }}
|
||||
{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- /* Shared with networkpolicy.yaml, which renders this Job's policy exactly
|
||||
when the Job itself renders. */}}
|
||||
{{- /* trim: include returns a string, and a whitespace-only one is truthy. */}}
|
||||
{{- $commands := include "seaweedfs.volumeResizeHookCommands" . | trim }}
|
||||
|
||||
|
||||
{{- if .Values.volume.resizeHook.enabled }}
|
||||
{{- $commands := list }}
|
||||
{{- range $vname, $volume := $volumes }}
|
||||
{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }}
|
||||
{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }}
|
||||
|
||||
{{- if $volume.enabled }}
|
||||
{{- $replicas := int $volume.replicas -}}
|
||||
{{- $statefulsetName := printf "%s-%s" $seaweedfsName $volumeName -}}
|
||||
{{- $statefulset := (lookup "apps/v1" "StatefulSet" $.Release.Namespace $statefulsetName) -}}
|
||||
|
||||
{{/* Check for changes in volumeClaimTemplates */}}
|
||||
{{- if $statefulset }}
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
{{- if eq .type "persistentVolumeClaim" }}
|
||||
{{- $desiredSize := .size }}
|
||||
{{- range $statefulset.spec.volumeClaimTemplates }}
|
||||
{{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) }}
|
||||
{{- $commands = append $commands (printf "kubectl delete statefulset %s --cascade=orphan" $statefulsetName) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/* Check for the need for patching existing PVCs */}}
|
||||
{{- range $dir := $volume.dataDirs }}
|
||||
{{- if eq .type "persistentVolumeClaim" }}
|
||||
{{- $desiredSize := .size }}
|
||||
{{- range $i, $e := until $replicas }}
|
||||
{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }}
|
||||
{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }}
|
||||
{{- if and $currentPVC }}
|
||||
{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }}
|
||||
{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }}
|
||||
{{- if gt $newSize $oldSize }}
|
||||
{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{- if $commands }}
|
||||
{{- if $commands }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
@@ -82,9 +38,7 @@ spec:
|
||||
command: ["sh", "-xec"]
|
||||
args:
|
||||
- |
|
||||
{{- range $commands }}
|
||||
{{ . }}
|
||||
{{- end }}
|
||||
{{ $commands | indent 14 }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
@@ -126,5 +80,4 @@ roleRef:
|
||||
kind: Role
|
||||
name: {{ $seaweedfsName }}-volume-resize-hook
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
Reference in New Issue
Block a user