diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index dc5566285..0ff81e9db 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -133,6 +133,23 @@ jobs: grep -A 12 "^kind: Ingress" /tmp/s3-ingress-labels.yaml | grep -q "^ external-dns: s3" echo "S3 ingress renders custom labels" + echo "=== Testing HTTPRoute ===" + helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \ + --set s3.enabled=true,s3.httpRoute.enabled=true \ + --set 's3.httpRoute.parentRefs[0].name=gateway' > /tmp/s3-httproute.yaml + grep -q "^kind: HTTPRoute" /tmp/s3-httproute.yaml + grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "name: test-seaweedfs-s3$" + grep -A 4 -- '- group: ""' /tmp/s3-httproute.yaml | grep -q "port: 8333$" + echo "S3 HTTPRoute routes to the s3 service by default" + helm template test $CHART_DIR --show-only templates/s3/s3-httproute.yaml \ + --set filer.s3.enabled=true,filer.s3.port=8334,s3.httpRoute.enabled=true > /tmp/filer-s3-httproute.yaml + grep -A 4 -- '- group: ""' /tmp/filer-s3-httproute.yaml | grep -q "port: 8334$" + echo "S3 on filer HTTPRoute uses filer.s3.port" + helm template test $CHART_DIR --show-only templates/master/master-httproute.yaml \ + --set allInOne.enabled=true,master.enabled=false,master.httpRoute.enabled=true > /tmp/allinone-master-httproute.yaml + grep -A 4 -- '- group: ""' /tmp/allinone-master-httproute.yaml | grep -q "name: test-seaweedfs-all-in-one$" + echo "All-in-one master HTTPRoute routes to the all-in-one service" + echo "=== Testing with all-in-one mode ===" helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index 8e9812f37..43f82499d 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -542,6 +542,26 @@ helm install seaweedfs-worker-vacuum seaweedfs/seaweedfs -f values-worker-vacuum helm install seaweedfs-worker-balance seaweedfs/seaweedfs -f values-worker-balance.yaml ``` +## Gateway API + +Every component with an `ingress` block (master, volume, filer, s3 and admin) can also be exposed through a [Gateway API](https://gateway-api.sigs.k8s.io/) `HTTPRoute`, for clusters that route through a Gateway instead of an Ingress controller. `.httpRoute` sits next to `.ingress` and is disabled by default. The chart does not create the Gateway, so point `parentRefs` at one that already exists, and the `gateway.networking.k8s.io/v1` CRDs must be installed. + +```yaml +s3: + httpRoute: + enabled: true + parentRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: my-gateway + namespace: gateway-system + sectionName: https + hostnames: + - s3.example.com +``` + +With `rules` left empty the route sends all traffic to the component's own Service and port. A rule may set `matches`, `filters`, `timeouts` and `backendRefs`, and a rule without `backendRefs` still routes to that Service. In all-in-one mode the master, volume, filer and s3 routes target the all-in-one Service. When S3 runs only on the filer (`filer.s3.enabled` without `s3.enabled`), the s3 route uses `filer.s3.port`. + ## Network Policies In a namespace with a default-deny policy the install hangs: the components cannot resolve each other, and the post-install bucket hook waits on the master and filer until it gives up. `networkPolicy.enabled` renders one `NetworkPolicy` per component, selecting its pods by the standard `app.kubernetes.io/{name,instance,component}` labels and admitting traffic from the other pods of the release on the ports that component listens on. diff --git a/k8s/charts/seaweedfs/templates/admin/admin-httproute.yaml b/k8s/charts/seaweedfs/templates/admin/admin-httproute.yaml new file mode 100644 index 000000000..e54ea7802 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/admin/admin-httproute.yaml @@ -0,0 +1,55 @@ +{{- if and .Values.admin.enabled .Values.admin.httpRoute.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "seaweedfs.componentName" (list . "admin") }} + namespace: {{ .Release.Namespace }} + {{- with .Values.admin.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: admin + {{- with .Values.admin.httpRoute.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.admin.httpRoute.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.admin.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.admin.httpRoute.rules | default (list dict) }} + - backendRefs: + {{- with .backendRefs }} + {{- toYaml . | nindent 8 }} + {{- else }} + - group: "" + kind: Service + name: {{ include "seaweedfs.componentName" (list $ "admin") }} + namespace: {{ $.Release.Namespace }} + port: {{ $.Values.admin.port }} + weight: 1 + {{- end }} + {{- with .matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/filer/filer-httproute.yaml b/k8s/charts/seaweedfs/templates/filer/filer-httproute.yaml new file mode 100644 index 000000000..e655c1c19 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/filer/filer-httproute.yaml @@ -0,0 +1,57 @@ +{{- $filerEnabled := or .Values.filer.enabled .Values.allInOne.enabled }} +{{- if and $filerEnabled .Values.filer.httpRoute.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "filer")) .Values.allInOne.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "seaweedfs.componentName" (list . "filer") }} + namespace: {{ .Release.Namespace }} + {{- with .Values.filer.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: filer + {{- with .Values.filer.httpRoute.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.filer.httpRoute.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.filer.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.filer.httpRoute.rules | default (list dict) }} + - backendRefs: + {{- with .backendRefs }} + {{- toYaml . | nindent 8 }} + {{- else }} + - group: "" + kind: Service + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + port: {{ $.Values.filer.port }} + weight: 1 + {{- end }} + {{- with .matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/master/master-httproute.yaml b/k8s/charts/seaweedfs/templates/master/master-httproute.yaml new file mode 100644 index 000000000..ad6e685e8 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/master/master-httproute.yaml @@ -0,0 +1,57 @@ +{{- $masterEnabled := or .Values.master.enabled .Values.allInOne.enabled }} +{{- if and $masterEnabled .Values.master.httpRoute.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "master")) .Values.allInOne.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "seaweedfs.componentName" (list . "master") }} + namespace: {{ .Release.Namespace }} + {{- with .Values.master.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: master + {{- with .Values.master.httpRoute.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.master.httpRoute.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.master.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.master.httpRoute.rules | default (list dict) }} + - backendRefs: + {{- with .backendRefs }} + {{- toYaml . | nindent 8 }} + {{- else }} + - group: "" + kind: Service + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + port: {{ $.Values.master.port }} + weight: 1 + {{- end }} + {{- with .matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/s3/s3-httproute.yaml b/k8s/charts/seaweedfs/templates/s3/s3-httproute.yaml new file mode 100644 index 000000000..33f477709 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/s3/s3-httproute.yaml @@ -0,0 +1,64 @@ +{{- $s3Enabled := or .Values.s3.enabled (and .Values.filer.s3.enabled (not .Values.allInOne.enabled)) (and .Values.allInOne.enabled .Values.allInOne.s3.enabled) }} +{{- if and $s3Enabled .Values.s3.httpRoute.enabled }} +{{- $serviceName := include "seaweedfs.componentName" (list . "s3") }} +{{- $s3Port := .Values.filer.s3.port }} +{{- if and .Values.allInOne.enabled .Values.allInOne.s3.enabled }} +{{- $serviceName = include "seaweedfs.componentName" (list . "all-in-one") }} +{{- $s3Port = .Values.allInOne.s3.port | default .Values.s3.port }} +{{- else if .Values.s3.enabled }} +{{- $s3Port = .Values.s3.port }} +{{- end }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "seaweedfs.componentName" (list . "s3") }} + namespace: {{ .Release.Namespace }} + {{- with .Values.s3.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: s3 + {{- with .Values.s3.httpRoute.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.s3.httpRoute.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.s3.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.s3.httpRoute.rules | default (list dict) }} + - backendRefs: + {{- with .backendRefs }} + {{- toYaml . | nindent 8 }} + {{- else }} + - group: "" + kind: Service + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + port: {{ $s3Port }} + weight: 1 + {{- end }} + {{- with .matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml b/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml new file mode 100644 index 000000000..0fe815878 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/volume/volume-httproute.yaml @@ -0,0 +1,57 @@ +{{- $volumeEnabled := or .Values.volume.enabled .Values.allInOne.enabled }} +{{- if and $volumeEnabled .Values.volume.httpRoute.enabled }} +{{- $serviceName := ternary (include "seaweedfs.componentName" (list . "all-in-one")) (include "seaweedfs.componentName" (list . "volume")) .Values.allInOne.enabled }} +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: {{ include "seaweedfs.componentName" (list . "volume") }} + namespace: {{ .Release.Namespace }} + {{- with .Values.volume.httpRoute.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: volume + {{- with .Values.volume.httpRoute.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- with .Values.volume.httpRoute.parentRefs }} + parentRefs: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.volume.httpRoute.hostnames }} + hostnames: + {{- toYaml . | nindent 4 }} + {{- end }} + rules: + {{- range .Values.volume.httpRoute.rules | default (list dict) }} + - backendRefs: + {{- with .backendRefs }} + {{- toYaml . | nindent 8 }} + {{- else }} + - group: "" + kind: Service + name: {{ $serviceName }} + namespace: {{ $.Release.Namespace }} + port: {{ $.Values.volume.port }} + weight: 1 + {{- end }} + {{- with .matches }} + matches: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .filters }} + filters: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .timeouts }} + timeouts: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 1a6315d05..ea10064de 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -311,6 +311,24 @@ master: # sub_filter_once off; tls: [] + # Gateway API HTTPRoute, an alternative to the ingress above. + httpRoute: + enabled: false + labels: {} + annotations: {} + # Gateways this route attaches to, for example: + # - group: gateway.networking.k8s.io + # kind: Gateway + # name: my-gateway + # namespace: gateway-system + # sectionName: https + parentRefs: [] + hostnames: [] + # Each rule may set matches, filters, timeouts and backendRefs. + # A rule without backendRefs routes to the master service. + # An empty list routes all traffic there. + rules: [] + extraEnvironmentVars: WEED_MASTER_VOLUME_GROWTH_COPY_1: "7" WEED_MASTER_VOLUME_GROWTH_COPY_2: "6" @@ -647,6 +665,24 @@ volume: # sub_filter '/seaweedfsstatic' './seaweedfsstatic'; # sub_filter_once off; + # Gateway API HTTPRoute, an alternative to the ingress above. + httpRoute: + enabled: false + labels: {} + annotations: {} + # Gateways this route attaches to, for example: + # - group: gateway.networking.k8s.io + # kind: Gateway + # name: my-gateway + # namespace: gateway-system + # sectionName: https + parentRefs: [] + hostnames: [] + # Each rule may set matches, filters, timeouts and backendRefs. + # A rule without backendRefs routes to the volume server service. + # An empty list routes all traffic there. + rules: [] + # Map of named volume groups for topology-aware deployments. # Each key inherits all fields from the `volume` section but can override # them locally—for example, replicas, nodeSelector, dataCenter, etc. @@ -939,6 +975,24 @@ filer: # requires ingress-nginx to run with --enable-ssl-passthrough. tls: [] + # Gateway API HTTPRoute, an alternative to the ingress above. + httpRoute: + enabled: false + labels: {} + annotations: {} + # Gateways this route attaches to, for example: + # - group: gateway.networking.k8s.io + # kind: Gateway + # name: my-gateway + # namespace: gateway-system + # sectionName: https + parentRefs: [] + hostnames: [] + # Each rule may set matches, filters, timeouts and backendRefs. + # A rule without backendRefs routes to the filer service. + # An empty list routes all traffic there. + rules: [] + # extraEnvVars is a list of extra environment variables to set with the stateful set. extraEnvironmentVars: # the WEED_MYSQL_* keys and the db credential secret only render while this is "true" @@ -1264,6 +1318,24 @@ s3: annotations: {} tls: [] + # Gateway API HTTPRoute, an alternative to the ingress above. + httpRoute: + enabled: false + labels: {} + annotations: {} + # Gateways this route attaches to, for example: + # - group: gateway.networking.k8s.io + # kind: Gateway + # name: my-gateway + # namespace: gateway-system + # sectionName: https + parentRefs: [] + hostnames: [] + # Each rule may set matches, filters, timeouts and backendRefs. + # A rule without backendRefs routes to the s3 service. + # An empty list routes all traffic there. + rules: [] + # Service settings service: type: ClusterIP @@ -1601,6 +1673,24 @@ admin: annotations: {} tls: [] + # Gateway API HTTPRoute, an alternative to the ingress above. + httpRoute: + enabled: false + labels: {} + annotations: {} + # Gateways this route attaches to, for example: + # - group: gateway.networking.k8s.io + # kind: Gateway + # name: my-gateway + # namespace: gateway-system + # sectionName: https + parentRefs: [] + hostnames: [] + # Each rule may set matches, filters, timeouts and backendRefs. + # A rule without backendRefs routes to the admin service. + # An empty list routes all traffic there. + rules: [] + service: type: ClusterIP annotations: {}