diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index d195b4a40..9276a4a0e 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -921,10 +921,8 @@ jobs: if comp not in pols: failed.append(f"{comp}: workload has no NetworkPolicy") for comp in pols: - # The hook Jobs are covered too, and the resize hook Job only renders - # when a cluster lookup says a PVC needs growing, so it is never in the - # rendered set here. - if comp not in wls and comp not in ("bucket-hook", "volume-resize-hook"): + # The bucket hook Job is covered too, and a Job is not a workload here. + if comp not in wls and comp != "bucket-hook": failed.append(f"{comp}: NetworkPolicy selects a component that is not deployed") # The ports a component listens on come from the same values as its @@ -1077,22 +1075,23 @@ jobs: # The API server rule goes only to the components that talk to it. apiserver = {c for c, p in pols.items() if any("ipBlock" in t for r in p["spec"]["egress"] for t in r.get("to") or [])} - expected = {"admin", "objectstorage-provisioner", "volume-resize-hook"} + expected = {"admin", "objectstorage-provisioner"} if apiserver != expected: failed.append(f"API server egress granted to {sorted(apiserver)}, expected {sorted(expected)}") else: print(f"API server egress limited to {sorted(expected)}") - # The resize hook runs as a pre-install hook at weight 0, before the release - # manifest is applied, so its policy has to be a hook itself and has to sort - # ahead of the Job. - rh = pols["volume-resize-hook"]["metadata"].get("annotations", {}) - if rh.get("helm.sh/hook") != "pre-install,pre-upgrade": - failed.append(f"volume-resize-hook policy is not a pre-install hook: {rh}") - elif int(rh.get("helm.sh/hook-weight", 0)) >= 0: - failed.append(f"volume-resize-hook policy weight {rh.get('helm.sh/hook-weight')} does not sort before the Job at 0") + # The resize hook's policy is gated on the same lookup as its Job, so + # neither is ever in a rendered set - a policy on its own would be an + # orphaned hook resource on every install, since Helm does not collect + # those. The hook annotations it carries when the lookup does hit are + # only reachable against a live cluster. + if "volume-resize-hook" in pols: + failed.append("volume-resize-hook policy rendered without its Job; the two " + "are gated on seaweedfs.volumeResizeHookCommands and have to " + "appear together") else: - print("volume-resize-hook policy is a pre-install hook ahead of the Job") + print("volume-resize-hook policy tracks its Job rather than rendering always") # The bucket hook is post-install, so the release manifest is already applied; # its policy must be a plain resource that uninstall cleans up. diff --git a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl index a27bb0aa5..044f6d455 100644 --- a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl @@ -360,6 +360,65 @@ true {{- end -}} {{- end -}} +{{/* The kubectl commands the volume resize hook has to run, one per line: a + cascade-orphan delete for every StatefulSet whose volumeClaimTemplates no + longer match the values, and a patch for every PVC the values grew. + + Empty when there is nothing to resize, which is what gates the Job. Read by + its NetworkPolicy too, which has to appear exactly when the Job does - a Job + without its policy hangs in a default-deny namespace, and a policy without + its Job is an orphaned hook resource on every install. + + Built on lookup, so it is always empty under helm template and on a fresh + install, where there is no StatefulSet to compare against yet. */}} +{{- define "seaweedfs.volumeResizeHookCommands" -}} +{{- $seaweedfsName := include "seaweedfs.fullname" $ }} +{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }} +{{- $commands := list }} +{{- if .Values.volume.resizeHook.enabled }} +{{- range $vname, $volume := $volumes }} +{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }} +{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }} +{{- if $volume.enabled }} +{{- $replicas := int $volume.replicas }} +{{- $statefulsetName := printf "%s-%s" $seaweedfsName $volumeName }} +{{- $statefulset := (lookup "apps/v1" "StatefulSet" $.Release.Namespace $statefulsetName) }} +{{- /* Check for changes in volumeClaimTemplates */}} +{{- if $statefulset }} +{{- range $dir := $volume.dataDirs }} +{{- if eq .type "persistentVolumeClaim" }} +{{- $desiredSize := .size }} +{{- range $statefulset.spec.volumeClaimTemplates }} +{{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) }} +{{- $commands = append $commands (printf "kubectl delete statefulset %s --cascade=orphan" $statefulsetName) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- /* Check for the need for patching existing PVCs */}} +{{- range $dir := $volume.dataDirs }} +{{- if eq .type "persistentVolumeClaim" }} +{{- $desiredSize := .size }} +{{- range $i, $e := until $replicas }} +{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }} +{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }} +{{- if $currentPVC }} +{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }} +{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }} +{{- if gt $newSize $oldSize }} +{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- join "\n" $commands }} +{{- end -}} + {{/* S3 TLS cert/key arguments, using custom secret if s3.tlsSecret is set */}} {{- define "seaweedfs.s3.tlsArgs" -}} {{- $prefix := .prefix -}} diff --git a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml index bb6bbabae..a79f05e7f 100644 --- a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml +++ b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml @@ -39,7 +39,6 @@ {{- $targets = append $targets (dict "component" "master" "ports" $ports) }} {{- end }} {{- /* Every volume group carries its own component label and its own ports. */}} -{{- $anyVolume := false }} {{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }} {{- range $vname, $volume := $volumes }} {{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }} @@ -50,7 +49,6 @@ {{- $ports = append $ports $volume.metricsPort }} {{- end }} {{- $targets = append $targets (dict "component" $volumeName "ports" $ports) }} -{{- $anyVolume = true }} {{- end }} {{- end }} {{- if .Values.filer.enabled }} @@ -115,11 +113,13 @@ {{- if include "seaweedfs.bucketHookEnabled" . }} {{- $targets = append $targets (dict "component" "bucket-hook" "ports" (list)) }} {{- end }} -{{- /* Whether the resize hook Job materialises depends on a cluster lookup, so - there is nothing to key on at render time. Its policy is created whenever - the hook is enabled and selects no pods on the upgrades that need no - resize. */}} -{{- if and .Values.volume.resizeHook.enabled $anyVolume }} +{{- /* Same cluster lookup the Job is gated on, so the policy appears on exactly + the upgrades that grow a PVC and never on the ones that do not. Helm does + not garbage-collect hook resources, so a policy rendered unconditionally + would be left in the namespace by every install that uninstalls again - + and would make egress.kubeApiServer.cidrs mandatory for every release, + for a Job most of them never run. */}} +{{- if include "seaweedfs.volumeResizeHookCommands" . | trim }} {{- $targets = append $targets (dict "component" "volume-resize-hook" "ports" (list) "apiserver" true "hook" "pre-install,pre-upgrade" "hookWeight" "-10") }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/volume/volume-resize-hook.yaml b/k8s/charts/seaweedfs/templates/volume/volume-resize-hook.yaml index f4d14969d..caa5cd430 100644 --- a/k8s/charts/seaweedfs/templates/volume/volume-resize-hook.yaml +++ b/k8s/charts/seaweedfs/templates/volume/volume-resize-hook.yaml @@ -1,54 +1,10 @@ {{- $seaweedfsName := include "seaweedfs.fullname" $ }} -{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }} +{{- /* Shared with networkpolicy.yaml, which renders this Job's policy exactly + when the Job itself renders. */}} +{{- /* trim: include returns a string, and a whitespace-only one is truthy. */}} +{{- $commands := include "seaweedfs.volumeResizeHookCommands" . | trim }} - -{{- if .Values.volume.resizeHook.enabled }} -{{- $commands := list }} -{{- range $vname, $volume := $volumes }} -{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }} -{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }} - -{{- if $volume.enabled }} -{{- $replicas := int $volume.replicas -}} -{{- $statefulsetName := printf "%s-%s" $seaweedfsName $volumeName -}} -{{- $statefulset := (lookup "apps/v1" "StatefulSet" $.Release.Namespace $statefulsetName) -}} - -{{/* Check for changes in volumeClaimTemplates */}} -{{- if $statefulset }} -{{- range $dir := $volume.dataDirs }} -{{- if eq .type "persistentVolumeClaim" }} -{{- $desiredSize := .size }} -{{- range $statefulset.spec.volumeClaimTemplates }} -{{- if and (eq .metadata.name $dir.name) (ne .spec.resources.requests.storage $desiredSize) }} -{{- $commands = append $commands (printf "kubectl delete statefulset %s --cascade=orphan" $statefulsetName) }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - -{{/* Check for the need for patching existing PVCs */}} -{{- range $dir := $volume.dataDirs }} -{{- if eq .type "persistentVolumeClaim" }} -{{- $desiredSize := .size }} -{{- range $i, $e := until $replicas }} -{{- $pvcName := printf "%s-%s-%s-%d" $dir.name $seaweedfsName $volumeName $e }} -{{- $currentPVC := (lookup "v1" "PersistentVolumeClaim" $.Release.Namespace $pvcName) }} -{{- if and $currentPVC }} -{{- $oldSize := include "seaweedfs.resource-quantity" $currentPVC.spec.resources.requests.storage }} -{{- $newSize := include "seaweedfs.resource-quantity" $desiredSize }} -{{- if gt $newSize $oldSize }} -{{- $commands = append $commands (printf "kubectl patch pvc %s-%s-%s-%d -p '{\"spec\":{\"resources\":{\"requests\":{\"storage\":\"%s\"}}}}'" $dir.name $seaweedfsName $volumeName $e $desiredSize) }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} -{{- end }} - -{{- end }} -{{- end }} - -{{- if $commands }} +{{- if $commands }} apiVersion: batch/v1 kind: Job metadata: @@ -82,9 +38,7 @@ spec: command: ["sh", "-xec"] args: - | - {{- range $commands }} - {{ . }} - {{- end }} +{{ $commands | indent 14 }} --- apiVersion: v1 kind: ServiceAccount @@ -126,5 +80,4 @@ roleRef: kind: Role name: {{ $seaweedfsName }}-volume-resize-hook apiGroup: rbac.authorization.k8s.io -{{- end }} {{- end }}