From 04e135c21e2fb5d20fca4101e3085b22d62450bf Mon Sep 17 00:00:00 2001 From: myooken Date: Thu, 19 Nov 2026 03:49:35 +0900 Subject: [PATCH] fix(helm): separate volume DNS discovery from ingress traffic (#12719) * helm: set publishNotReadyAddresses on the volume headless service * fix(helm): separate volume DNS discovery from ingress traffic * test(helm): cover volume DNS discovery and ingress routing * fix(helm): rename the volume routing Service so volumes.client cannot collide * docs(helm): name the ready-only Service for explicit volume backendRefs Co-Authored-By: Claude Opus 5.5 * fix(helm): fail when the volume routing Service name collides after truncation Co-Authored-By: Claude Opus 5.5 * docs(helm): point explicit volume backendRefs at the rendered client-volume name Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Chris Lu Co-authored-by: Claude Opus 5.5 --- k8s/charts/seaweedfs/README.md | 2 + .../templates/volume/volume-httproute.yaml | 2 +- .../templates/volume/volume-ingress.yaml | 2 +- .../volume/volume-service-client.yaml | 42 +++++++ .../templates/volume/volume-service.yaml | 3 + .../seaweedfs/tests/volume_service_test.go | 107 ++++++++++++++++++ 6 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 k8s/charts/seaweedfs/templates/volume/volume-service-client.yaml create mode 100644 k8s/charts/seaweedfs/tests/volume_service_test.go diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index eff298e74a..f294c255a8 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -569,6 +569,8 @@ s3: With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`. +Outside all-in-one mode, the volume Ingress and the default volume route target the `client-volume` Service, which lists only Ready volume pods. The `volume` Service also publishes pods that are not Ready yet, so their DNS records can exist before readiness. An explicit `backendRefs` entry that names the `volume` Service can therefore reach a volume server before it is Ready. Name the `client-volume` Service instead, with `volume.port`, and copy its `metadata.name` from `helm template` because long release names are truncated. + ## Network Policies In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on. diff --git a/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml b/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml index 1880a83cf2..9635731283 100644 --- a/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml +++ b/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml @@ -1,6 +1,6 @@ {{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }} {{- if and $volumeEnabled .Values.volume.httpRoute.enabled }} -{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }} apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: diff --git a/k8s/charts/seaweedfs/templates/volume/volume-ingress.yaml b/k8s/charts/seaweedfs/templates/volume/volume-ingress.yaml index 6d8b5369de..4a8a25286a 100644 --- a/k8s/charts/seaweedfs/templates/volume/volume-ingress.yaml +++ b/k8s/charts/seaweedfs/templates/volume/volume-ingress.yaml @@ -2,7 +2,7 @@ {{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }} {{- if and $volumeEnabled .Values.volume.ingress.enabled }} {{- /* Determine service name based on deployment mode */}} -{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "client-volume")) .Values.allInOne.enabled }} {{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }} apiVersion: networking.k8s.io/v1 {{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }} diff --git a/k8s/charts/seaweedfs/templates/volume/volume-service-client.yaml b/k8s/charts/seaweedfs/templates/volume/volume-service-client.yaml new file mode 100644 index 0000000000..97b0562333 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/volume/volume-service-client.yaml @@ -0,0 +1,42 @@ +{{- /* Ready-only backend for the volume ingress and HTTPRoute; the volume Service publishes not-ready pods for DNS. + Not named volume-client: every volume- name belongs to a group under .Values.volumes. */}} +{{- if and .Values.volume.enabled (not .Values.allInOne.enabled) (or .Values.volume.ingress.enabled .Values.volume.httpRoute.enabled) }} +{{- $name := include "seaweedfs.componentName" (list . "client-volume") }} +{{- $volumes := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }} +{{- range $vname, $volume := $volumes }} +{{- $volumeName := trimSuffix "-" (printf "volume-%s" $vname) }} +{{- $volume := mergeOverwrite (deepCopy $.Values.volume) (dict "enabled" true) $volume }} +{{- if and $volume.enabled (eq (include "seaweedfs.componentName" (list $ $volumeName)) $name) }} +{{- fail (printf "The volume Ingress/HTTPRoute Service %q has the same name as the %s Service after truncation. Shorten fullnameOverride or the release name." $name $volumeName) }} +{{- end }} +{{- end }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }} +{{- if .Values.volume.annotations }} + annotations: + {{- toYaml .Values.volume.annotations | nindent 4 }} +{{- end }} +spec: + clusterIP: None + internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }} + {{- /* No metrics port, so the volume ServiceMonitor does not scrape each pod twice */}} + ports: + - name: "swfs-volume" + port: {{ .Values.volume.port }} + targetPort: {{ .Values.volume.port }} + protocol: TCP + selector: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/volume/volume-service.yaml b/k8s/charts/seaweedfs/templates/volume/volume-service.yaml index 0840ccfb9a..38576dc368 100644 --- a/k8s/charts/seaweedfs/templates/volume/volume-service.yaml +++ b/k8s/charts/seaweedfs/templates/volume/volume-service.yaml @@ -23,6 +23,9 @@ metadata: {{- end }} spec: clusterIP: None + {{- /* The master hands out ${POD_NAME}. as soon as a volume server heartbeats, before it is Ready. + Ingress and HTTPRoute use the client-volume Service instead, which publishes only Ready pods. */}} + publishNotReadyAddresses: true internalTrafficPolicy: {{ $volume.internalTrafficPolicy | default "Cluster" }} ports: - name: "swfs-volume" diff --git a/k8s/charts/seaweedfs/tests/volume_service_test.go b/k8s/charts/seaweedfs/tests/volume_service_test.go new file mode 100644 index 0000000000..fe3dd89345 --- /dev/null +++ b/k8s/charts/seaweedfs/tests/volume_service_test.go @@ -0,0 +1,107 @@ +package tests + +import ( + "fmt" + "os/exec" + "strings" + "testing" +) + +func findDocument(documents []map[string]interface{}, kind, name string) map[string]interface{} { + for _, document := range documents { + if document["kind"] == kind && document["metadata"].(map[string]interface{})["name"] == name { + return document + } + } + return nil +} + +func TestVolumeServicePublishesNotReadyPods(t *testing.T) { + _, documents := render(t) + service := findDocument(documents, "Service", "vector-test-seaweedfs-volume") + if service == nil { + t.Fatal("no volume service") + } + if service["spec"].(map[string]interface{})["publishNotReadyAddresses"] != true { + t.Fatal("volume pod DNS names must resolve before the pods are Ready") + } + if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil { + t.Fatal("client-volume service rendered without ingress or httpRoute") + } +} + +func TestVolumeIngressUsesReadyOnlyService(t *testing.T) { + _, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volume.metricsPort=9327") + client := findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") + if client == nil { + t.Fatal("no client-volume service") + } + spec := client["spec"].(map[string]interface{}) + if _, found := spec["publishNotReadyAddresses"]; found { + t.Fatal("client-volume must publish only Ready pods") + } + for _, port := range spec["ports"].([]interface{}) { + if port.(map[string]interface{})["name"] == "metrics" { + t.Fatal("client-volume must not expose metrics, or the ServiceMonitor scrapes each pod twice") + } + } + + ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume") + if ingress == nil { + t.Fatal("no volume ingress") + } + path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{}) + if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" { + t.Fatalf("volume ingress backend is %v", backend) + } + + route := findDocument(documents, "HTTPRoute", "vector-test-seaweedfs-volume") + if route == nil { + t.Fatal("no volume httpRoute") + } + rule := route["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{}) + if backend := rule["backendRefs"].([]interface{})[0].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-client-volume" { + t.Fatalf("volume httpRoute backend is %v", backend) + } +} + +func TestVolumeRoutingServiceDoesNotCollideWithVolumeGroups(t *testing.T) { + _, documents := render(t, "volume.ingress.enabled=true", "volume.httpRoute.enabled=true", "volumes.client.replicas=1") + seen := map[string]bool{} + for _, document := range documents { + key := fmt.Sprint(document["kind"], "/", document["metadata"].(map[string]interface{})["name"]) + if seen[key] { + t.Fatalf("%s rendered twice", key) + } + seen[key] = true + } + if !seen["Service/vector-test-seaweedfs-volume-client"] || !seen["Service/vector-test-seaweedfs-client-volume"] { + t.Fatal("volumes.client group Service and the routing Service must both render") + } +} + +func TestVolumeRoutingServiceRejectsTruncatedCollision(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + fullname := strings.Repeat("a", 49) + "-client" + output, err := exec.Command("helm", "template", "test", "..", "--set", "fullnameOverride="+fullname, "--set", "volume.ingress.enabled=true").CombinedOutput() + if err == nil || !strings.Contains(string(output), "same name as the volume Service") { + t.Fatalf("colliding routing Service accepted: %v %s", err, output) + } +} + +func TestVolumeIngressAllInOne(t *testing.T) { + _, documents := render(t, "allInOne.enabled=true", "volume.ingress.enabled=true") + if findDocument(documents, "Service", "vector-test-seaweedfs-client-volume") != nil { + t.Fatal("client-volume service rendered in all-in-one mode") + } + ingress := findDocument(documents, "Ingress", "ingress-vector-test-seaweedfs-volume") + if ingress == nil { + t.Fatal("no volume ingress") + } + path := ingress["spec"].(map[string]interface{})["rules"].([]interface{})[0].(map[string]interface{})["http"].(map[string]interface{})["paths"].([]interface{})[0].(map[string]interface{}) + if backend := path["backend"].(map[string]interface{})["service"].(map[string]interface{})["name"]; backend != "vector-test-seaweedfs-all-in-one" { + t.Fatalf("all-in-one volume ingress backend is %v", backend) + } +}