From 5abb18e678571519ed77e0c1bb1947cfe20b670e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 9 Nov 2026 16:34:57 -0800 Subject: [PATCH] helm: manage vector engines and provision vector buckets (#12518) * helm: deploy vector engines with TLS, probes, and monitoring * helm: provision vector buckets through the S3 Vectors API * helm: bracket IPv6 pod addresses for vector listeners * helm: discover vector engines through the all-in-one master * helm: accept vector component NetworkPolicy overrides * helm: bind vector metrics to the pod address on both IP families * helm: provision vector buckets through embedded and all-in-one gateways * docs: use the full Helm name for external vector certificates * helm: render vector instance labels as strings * helm: allow vector bucket hooks to use restricted security contexts * helm: honor vector security context enable switches * helm: prevent duplicate vector monitoring labels * helm: reject metrics flags with the default pre-metrics vector image * test: skip vector chart compatibility checks when Helm is unavailable --- .github/workflows/helm_ci.yml | 7 + k8s/charts/seaweedfs/Chart.yaml | 2 +- k8s/charts/seaweedfs/README.md | 44 +++++ .../seaweedfs/templates/cert/vector-cert.yaml | 50 ++++++ .../seaweedfs/templates/shared/_helpers.tpl | 12 ++ .../templates/shared/networkpolicy.yaml | 14 +- .../templates/shared/security-configmap.yaml | 4 + .../templates/vector/vector-bucket-hook.yaml | 167 ++++++++++++++++++ .../templates/vector/vector-deployment.yaml | 166 +++++++++++++++++ .../templates/vector/vector-service.yaml | 22 +++ .../vector/vector-servicemonitor.yaml | 25 +++ .../seaweedfs/tests/vector_bucket_test.go | 113 ++++++++++++ k8s/charts/seaweedfs/tests/vector_test.go | 148 ++++++++++++++++ k8s/charts/seaweedfs/values.yaml | 54 ++++++ 14 files changed, 826 insertions(+), 2 deletions(-) create mode 100644 k8s/charts/seaweedfs/templates/cert/vector-cert.yaml create mode 100644 k8s/charts/seaweedfs/templates/vector/vector-bucket-hook.yaml create mode 100644 k8s/charts/seaweedfs/templates/vector/vector-deployment.yaml create mode 100644 k8s/charts/seaweedfs/templates/vector/vector-service.yaml create mode 100644 k8s/charts/seaweedfs/templates/vector/vector-servicemonitor.yaml create mode 100644 k8s/charts/seaweedfs/tests/vector_bucket_test.go create mode 100644 k8s/charts/seaweedfs/tests/vector_test.go diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index d9e8f70bc1..98e993405e 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -37,6 +37,13 @@ jobs: with: version: v3.18.4 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Test vector chart resources + run: go test ./k8s/charts/seaweedfs/tests -count=1 + - uses: actions/setup-python@v7 with: python-version: '3.10' diff --git a/k8s/charts/seaweedfs/Chart.yaml b/k8s/charts/seaweedfs/Chart.yaml index 1f32d040e4..d17570f1ab 100644 --- a/k8s/charts/seaweedfs/Chart.yaml +++ b/k8s/charts/seaweedfs/Chart.yaml @@ -3,4 +3,4 @@ description: SeaweedFS name: seaweedfs appVersion: "4.53" # Dev note: Trigger a helm chart release by `git tag -a helm-` -version: 4.53.0 +version: 4.53.1 diff --git a/k8s/charts/seaweedfs/README.md b/k8s/charts/seaweedfs/README.md index 02139d8989..463f3a3771 100644 --- a/k8s/charts/seaweedfs/README.md +++ b/k8s/charts/seaweedfs/README.md @@ -687,3 +687,47 @@ generates a new UUID and the license stops matching. See [Master data](#master-data-hostpath-vs-a-claim). Check the binding with `kubectl exec -- curl -s localhost:9333/license/status` (`cluster_uuid` must equal `license_uuid`). + + +## Vector buckets + +The S3 gateway serves the S3 Vectors API on its existing endpoint. Enable a vector engine and Lance maintenance worker alongside the cluster: + +```yaml +s3: + enabled: true +vector: + enabled: true + replicas: 1 +worker: + enabled: true +admin: + enabled: true + secret: + adminPassword: change-me +global: + seaweedfs: + securityConfig: + jwtSigning: + filerWrite: true +``` + +The engine registers its pod address with the masters and stores indexes through the filers and volume servers. Both Go and Rust volume servers use the same setup. The maintenance worker builds and updates ANN indexes; queries can scan data before indexing completes. Enabling filer write signing also requires clients writing directly to the filer HTTP API to supply a signed JWT. + +The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverride` to select a compatible engine build, `vector.masters` for an external cluster, and `vector.resources` to set memory and CPU limits. The cache defaults to 512 MiB and is configured with `vector.readCacheMB`. + +`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets. + +The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports. + + +To provision vector buckets during installation and upgrades, set: + +```yaml +vector: + enabled: true + createBuckets: + - embeddings +``` + +The hook uses the AWS CLI's S3 Vectors API and works with existing SeaweedFS vector releases. It creates missing buckets, preserves existing vector buckets, and fails if the name belongs to an Iceberg bucket. It uses the chart's generated S3 admin credentials, including `s3.credentials.admin.existingSecret`. With a custom `s3.existingConfigSecret` or an external endpoint, set `vector.bucketHook.credentialsSecret` to a Secret containing `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and optionally `AWS_SESSION_TOKEN`). Key names are configurable. `vector.bucketHook.endpoint` overrides the internal S3 service URL; `caBundleSecret` and `caBundleKey` configure a private HTTPS CA. The hook image and resource limits are configurable under `vector.bucketHook`. For restricted Pod Security, select a non-root-compatible image and configure `vector.bucketHook.podSecurityContext` and `containerSecurityContext`, including `runAsNonRoot` and the image user ID. diff --git a/k8s/charts/seaweedfs/templates/cert/vector-cert.yaml b/k8s/charts/seaweedfs/templates/cert/vector-cert.yaml new file mode 100644 index 0000000000..d82fbc0619 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/cert/vector-cert.yaml @@ -0,0 +1,50 @@ +{{- include "seaweedfs.compat" . -}} +{{- if and .Values.vector.enabled .Values.global.seaweedfs.enableSecurity (not .Values.certificates.externalCertificates.enabled)}} +apiVersion: cert-manager.io/v1{{ if .Values.global.seaweedfs.certificates.alphacrds }}alpha1{{ end }} +kind: Certificate +metadata: + name: {{ include "seaweedfs.fullname" . }}-vector-cert + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + app.kubernetes.io/name: {{ template "seaweedfs.name" . }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/component: vector + {{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }} + {{- if .Values.vector.annotations }} + annotations: + {{- toYaml .Values.vector.annotations | nindent 4 }} + {{- end }} +spec: + secretName: {{ include "seaweedfs.fullname" . }}-vector-cert + issuerRef: + name: {{ include "seaweedfs.fullname" . }}-ca-issuer + kind: Issuer + commonName: {{ .Values.certificates.commonName }} + subject: + organizations: + - "SeaweedFS CA" + dnsNames: + - '*.{{ include "seaweedfs.fullname" . }}-vector' + - '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}' + - '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}.svc' + - '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}.svc.cluster.local' +{{- if .Values.certificates.dnsNames }} +{{- range .Values.certificates.dnsNames }} + - {{ . | quote }} +{{- end }} +{{- end }} +{{- if .Values.certificates.ipAddresses }} + ipAddresses: + {{- range .Values.certificates.ipAddresses }} + - {{ . }} + {{- end }} +{{- end }} + privateKey: + algorithm: {{ .Values.certificates.keyAlgorithm }} + size: {{ .Values.certificates.keySize }} + duration: {{ .Values.certificates.duration }} + renewBefore: {{ .Values.certificates.renewBefore }} +{{- end }} + diff --git a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl index 8df5ab1e0e..5d77663663 100644 --- a/k8s/charts/seaweedfs/templates/shared/_helpers.tpl +++ b/k8s/charts/seaweedfs/templates/shared/_helpers.tpl @@ -770,3 +770,15 @@ on install. {{- $context := merge (dict "Values" (omit .Values "commonLabels")) (omit . "Values") -}} {{- include (print .Template.BasePath "/master/master-configmap.yaml") $context | sha256sum -}} {{- end -}} + +{{- define "seaweedfs.vector.selectorLabels" -}} +app.kubernetes.io/name: {{ include "seaweedfs.name" . }} +app.kubernetes.io/instance: {{ .Release.Name | quote }} +app.kubernetes.io/component: vector +{{- end -}} +{{- define "seaweedfs.vector.labels" -}} +{{ include "seaweedfs.vector.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +{{- with include "seaweedfs.commonLabels" . }}{{ . | nindent 0 }}{{ end }} +{{- end -}} diff --git a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml index 4af04ea38a..5abbbe95ef 100644 --- a/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml +++ b/k8s/charts/seaweedfs/templates/shared/networkpolicy.yaml @@ -110,6 +110,18 @@ {{- $targets = append $targets (dict "component" "worker" "ports" $ports) }} {{- end }} +{{- if .Values.vector.enabled }} +{{- $ports := list .Values.vector.port }} +{{- if .Values.vector.metricsPort }} +{{- $ports = append $ports .Values.vector.metricsPort }} +{{- end }} +{{- $targets = append $targets (dict "component" "vector" "ports" $ports) }} +{{- end }} + +{{- if and .Values.vector.enabled .Values.vector.createBuckets }} +{{- $targets = append $targets (dict "component" "vector-bucket-hook" "ports" (list)) }} +{{- end }} + {{- if .Values.cosi.enabled }} {{- /* The provisioner sidecar watches the COSI CRs, so it needs the API server; nothing needs to reach the driver itself. */}} @@ -143,7 +155,7 @@ every component the chart can produce rather than the ones enabled here, so one values file can still carry overrides for a component this release leaves off. */}} -{{- $known := list "master" "filer" "s3" "sftp" "admin" "worker" "volume" "bucket-hook" "volume-resize-hook" "objectstorage-provisioner" "seaweedfs-all-in-one" }} +{{- $known := list "vector" "vector-bucket-hook" "master" "filer" "s3" "sftp" "admin" "worker" "volume" "bucket-hook" "volume-resize-hook" "objectstorage-provisioner" "seaweedfs-all-in-one" }} {{- range $vname, $_ := (.Values.volumes | default dict) }} {{- $known = append $known (trimSuffix "-" (printf "volume-%s" $vname)) }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/shared/security-configmap.yaml b/k8s/charts/seaweedfs/templates/shared/security-configmap.yaml index f5e7e4a67f..d1572ee7af 100644 --- a/k8s/charts/seaweedfs/templates/shared/security-configmap.yaml +++ b/k8s/charts/seaweedfs/templates/shared/security-configmap.yaml @@ -94,6 +94,10 @@ data: cert = "/usr/local/share/ca-certificates/admin/tls.crt" key = "/usr/local/share/ca-certificates/admin/tls.key" + [grpc.vector] + cert = "/usr/local/share/ca-certificates/vector/tls.crt" + key = "/usr/local/share/ca-certificates/vector/tls.key" + [grpc.worker] cert = "/usr/local/share/ca-certificates/worker/tls.crt" key = "/usr/local/share/ca-certificates/worker/tls.key" diff --git a/k8s/charts/seaweedfs/templates/vector/vector-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/vector/vector-bucket-hook.yaml new file mode 100644 index 0000000000..2d06ed8c86 --- /dev/null +++ b/k8s/charts/seaweedfs/templates/vector/vector-bucket-hook.yaml @@ -0,0 +1,167 @@ +{{- include "seaweedfs.compat" . -}} +{{- if and .Values.vector.enabled .Values.vector.createBuckets }} +{{- $hook := .Values.vector.bucketHook }} +{{- $component := "s3" }} +{{- $port := .Values.s3.port }} +{{- $enabled := .Values.s3.enabled }} +{{- $auth := .Values.s3.enableAuth }} +{{- $configSecret := .Values.s3.existingConfigSecret }} +{{- if and .Values.allInOne.enabled .Values.allInOne.s3.enabled }} +{{- $component = "all-in-one" }} +{{- $port = .Values.allInOne.s3.port | default .Values.s3.port }} +{{- $enabled = true }} +{{- $auth = or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }} +{{- $configSecret = or .Values.allInOne.s3.existingConfigSecret .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret }} +{{- else if and (not $enabled) .Values.filer.s3.enabled }} +{{- $component = "filer" }} +{{- $port = .Values.filer.s3.port }} +{{- $enabled = true }} +{{- $auth = .Values.filer.s3.enableAuth }} +{{- $configSecret = .Values.filer.s3.existingConfigSecret }} +{{- end }} +{{- if and (not $enabled) (not $hook.endpoint) }} +{{- fail "vector.createBuckets requires an enabled S3 gateway or vector.bucketHook.endpoint" }} +{{- end }} +{{- range .Values.vector.createBuckets }} +{{- if not (kindIs "string" .) }}{{ fail "vector.createBuckets entries must be bucket names" }}{{ end }} +{{- if not (and (ge (len .) 3) (le (len .) 63) (regexMatch "^[a-z0-9][a-z0-9-]*[a-z0-9]$" .)) }} +{{- fail "vector.createBuckets names must be 3-63 lowercase letters, digits or hyphens" }} +{{- end }} +{{- end }} +{{- $credentials := .Values.s3.credentials | default dict }} +{{- $admin := $credentials.admin | default dict }} +{{- $secret := $hook.credentialsSecret }} +{{- $accessKey := $hook.accessKeyKey }} +{{- $secretKey := $hook.secretKeyKey }} +{{- if and $auth (not $secret) }} +{{- if $configSecret }}{{ fail "vector.bucketHook.credentialsSecret is required with s3.existingConfigSecret" }}{{ end }} +{{- $secret = $admin.existingSecret | default (printf "%s-s3-secret" (include "seaweedfs.fullname" .)) }} +{{- $accessKey = $admin.accessKeyKey | default "admin_access_key_id" }} +{{- $secretKey = $admin.secretKeyKey | default "admin_secret_access_key" }} +{{- end }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "seaweedfs.fullname" . }}-vector-bucket-hook + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + app.kubernetes.io/name: {{ include "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/component: vector-bucket-hook + {{- with include "seaweedfs.commonLabels" . }}{{ . | nindent 4 }}{{ end }} + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: 2 + activeDeadlineSeconds: 360 + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "seaweedfs.name" . }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + app.kubernetes.io/component: vector-bucket-hook + spec: + restartPolicy: Never + automountServiceAccountToken: false + {{- with $hook.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.vector.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.vector.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} + containers: + - name: create-vector-buckets + image: {{ $hook.image | quote }} + command: ["/bin/sh", "-ec"] + args: + - | + for bucket in {{ range .Values.vector.createBuckets }}{{ . | quote }} {{ end }}; do + attempt=0 + while ! aws --endpoint-url "$S3_ENDPOINT" s3vectors get-vector-bucket --vector-bucket-name "$bucket" >/dev/null 2>/tmp/bucket-error; do + if grep -q NotFound /tmp/bucket-error; then + aws --endpoint-url "$S3_ENDPOINT" s3vectors create-vector-bucket --vector-bucket-name "$bucket" || + aws --endpoint-url "$S3_ENDPOINT" s3vectors get-vector-bucket --vector-bucket-name "$bucket" + break + fi + if grep -q AccessDenied /tmp/bucket-error; then cat /tmp/bucket-error >&2; exit 1; fi + attempt=$((attempt + 1)) + if [ "$attempt" -ge 60 ]; then cat /tmp/bucket-error >&2; exit 1; fi + sleep 5 + done + done + env: + - name: S3_ENDPOINT + value: {{ $hook.endpoint | default (printf "http://%s.%s:%d" (include "seaweedfs.componentName" (list . $component)) (include "seaweedfs.namespace" .) (int $port)) | quote }} + - name: AWS_DEFAULT_REGION + value: {{ $hook.region | quote }} + - name: AWS_EC2_METADATA_DISABLED + value: "true" + - name: AWS_PAGER + value: "" + - name: AWS_MAX_ATTEMPTS + value: "2" + - name: AWS_ACCESS_KEY_ID + {{- if $secret }} + valueFrom: + secretKeyRef: + name: {{ $secret | quote }} + key: {{ $accessKey | quote }} + {{- else }} + value: seaweedfs + {{- end }} + - name: AWS_SECRET_ACCESS_KEY + {{- if $secret }} + valueFrom: + secretKeyRef: + name: {{ $secret | quote }} + key: {{ $secretKey | quote }} + {{- else }} + value: seaweedfs + {{- end }} + {{- if $hook.credentialsSecret }} + - name: AWS_SESSION_TOKEN + valueFrom: + secretKeyRef: + name: {{ $secret | quote }} + key: {{ $hook.sessionTokenKey | quote }} + optional: true + {{- end }} + {{- if $hook.caBundleSecret }} + - name: AWS_CA_BUNDLE + value: /etc/ssl/vector/ca.crt + {{- end }} + securityContext: + {{- toYaml $hook.containerSecurityContext | nindent 12 }} + {{- with $hook.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if $hook.caBundleSecret }} + - name: ca + mountPath: /etc/ssl/vector + readOnly: true + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if $hook.caBundleSecret }} + - name: ca + secret: + secretName: {{ $hook.caBundleSecret | quote }} + items: + - key: {{ $hook.caBundleKey | quote }} + path: ca.crt + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/vector/vector-deployment.yaml b/k8s/charts/seaweedfs/templates/vector/vector-deployment.yaml new file mode 100644 index 0000000000..248abd66aa --- /dev/null +++ b/k8s/charts/seaweedfs/templates/vector/vector-deployment.yaml @@ -0,0 +1,166 @@ +{{- include "seaweedfs.compat" . -}} +{{- if .Values.vector.enabled }} +{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }} +{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }} +{{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "seaweedfs.componentName" (list . "vector") }} + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + {{- include "seaweedfs.vector.labels" . | nindent 4 }} + {{- with .Values.vector.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.vector.replicas }} + selector: + matchLabels: + {{- include "seaweedfs.vector.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "seaweedfs.vector.selectorLabels" . | nindent 8 }} + {{- with .Values.podLabels }}{{ toYaml . | nindent 8 }}{{ end }} + {{- with .Values.vector.podLabels }}{{ toYaml . | nindent 8 }}{{ end }} + annotations: + {{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }} + {{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }} + spec: + {{- include "seaweedfs.imagePullSecrets" . | nindent 6 }} + automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }} + {{- with .Values.vector.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.vector.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.vector.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and .Values.vector.podSecurityContext (or (not (hasKey .Values.vector.podSecurityContext "enabled")) .Values.vector.podSecurityContext.enabled) }} + securityContext: + {{- omit .Values.vector.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + containers: + - name: vector + image: {{ .Values.vector.imageOverride | default (include "seaweedfs.image" .) | quote }} + imagePullPolicy: {{ .Values.global.seaweedfs.imagePullPolicy }} + {{- $masters := include "seaweedfs.masterServerArg" . }} + {{- if .Values.allInOne.enabled }}{{ $masters = include "seaweedfs.cluster.masterAddress" . }}{{ end }} + command: ["/bin/sh", "-ec"] + args: + - 'case "$POD_IP" in *:*) POD_IP="[$POD_IP]";; esac; exec /usr/bin/weed-vector --listen="$POD_IP:{{ .Values.vector.port }}" "$@"' + - vector + - {{ printf "--masters=%s" (.Values.vector.masters | default $masters) | quote }} + - {{ printf "--read-cache-mb=%d" (int .Values.vector.readCacheMB) | quote }} + {{- if .Values.filer.filerGroup }} + - {{ printf "--filer-group=%s" .Values.filer.filerGroup | quote }} + {{- end }} + {{- if .Values.vector.nprobes }} + - {{ printf "--nprobes=%d" (int .Values.vector.nprobes) | quote }} + {{- end }} + {{- if include "seaweedfs.securityConfigEnabled" . }} + - --security-toml=/etc/seaweedfs/security.toml + {{- end }} + {{- if .Values.vector.metricsPort }} + - {{ printf "--metrics-port=%d" (int .Values.vector.metricsPort) | quote }} + - --metrics-ip=$(POD_IP) + {{- end }} + {{- range .Values.vector.extraArgs }} + - {{ . | quote }} + {{- end }} + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- $env := dict }} + {{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.vector "target" $env) }} + {{- range $key := keys $env | sortAlpha }} + - name: {{ $key }} + {{- $value := index $env $key }} + {{- if kindIs "map" $value }} + valueFrom: + {{- toYaml $value | nindent 16 }} + {{- else }} + value: {{ tpl (toString $value) $ | quote }} + {{- end }} + {{- end }} + ports: + - name: grpc + containerPort: {{ .Values.vector.port }} + {{- if .Values.vector.metricsPort }} + - name: metrics + containerPort: {{ .Values.vector.metricsPort }} + {{- end }} + startupProbe: + tcpSocket: + port: grpc + periodSeconds: 5 + failureThreshold: 60 + readinessProbe: + {{- if .Values.vector.metricsPort }} + httpGet: + path: /ready + port: metrics + {{- else }} + tcpSocket: + port: grpc + {{- end }} + periodSeconds: 5 + livenessProbe: + {{- if .Values.vector.metricsPort }} + httpGet: + path: /health + port: metrics + {{- else }} + tcpSocket: + port: grpc + {{- end }} + initialDelaySeconds: 10 + periodSeconds: 10 + {{- if or (not (hasKey .Values.vector.containerSecurityContext "enabled")) .Values.vector.containerSecurityContext.enabled }} + securityContext: + {{- omit .Values.vector.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + {{- with .Values.vector.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + {{- if include "seaweedfs.securityConfigEnabled" . }} + - name: security-config + mountPath: /etc/seaweedfs + readOnly: true + {{- end }} + {{- if .Values.global.seaweedfs.enableSecurity }} + {{- range list "ca" "client" "vector" }} + - name: {{ . }}-cert + mountPath: /usr/local/share/ca-certificates/{{ . }} + readOnly: true + {{- end }} + {{- end }} + volumes: + - name: tmp + emptyDir: {} + {{- if include "seaweedfs.securityConfigEnabled" . }} + - name: security-config + configMap: + name: {{ include "seaweedfs.fullname" . }}-security-config + {{- end }} + {{- if .Values.global.seaweedfs.enableSecurity }} + {{- range list "ca" "client" "vector" }} + - name: {{ . }}-cert + secret: + secretName: {{ include "seaweedfs.fullname" $ }}-{{ . }}-cert + {{- end }} + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/vector/vector-service.yaml b/k8s/charts/seaweedfs/templates/vector/vector-service.yaml new file mode 100644 index 0000000000..e0b0cf3b3a --- /dev/null +++ b/k8s/charts/seaweedfs/templates/vector/vector-service.yaml @@ -0,0 +1,22 @@ +{{- if .Values.vector.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "seaweedfs.componentName" (list . "vector") }} + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + {{- include "seaweedfs.vector.labels" . | nindent 4 }} +spec: + clusterIP: None + selector: + {{- include "seaweedfs.vector.selectorLabels" . | nindent 4 }} + ports: + - name: grpc + port: {{ .Values.vector.port }} + targetPort: grpc + {{- if .Values.vector.metricsPort }} + - name: metrics + port: {{ .Values.vector.metricsPort }} + targetPort: metrics + {{- end }} +{{- end }} diff --git a/k8s/charts/seaweedfs/templates/vector/vector-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/vector/vector-servicemonitor.yaml new file mode 100644 index 0000000000..b8eb818b5b --- /dev/null +++ b/k8s/charts/seaweedfs/templates/vector/vector-servicemonitor.yaml @@ -0,0 +1,25 @@ +{{- if and .Values.vector.enabled .Values.vector.metricsPort .Values.global.seaweedfs.monitoring.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "seaweedfs.componentName" (list . "vector") }} + namespace: {{ include "seaweedfs.namespace" . | quote }} + labels: + {{- include "seaweedfs.vector.selectorLabels" . | nindent 4 }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + {{- with include "seaweedfs.commonLabels" (dict "context" $ "exclude" $.Values.global.seaweedfs.monitoring.additionalLabels) }}{{ . | nindent 4 }}{{ end }} + {{- with .Values.global.seaweedfs.monitoring.additionalLabels }}{{ toYaml . | nindent 4 }}{{ end }} + {{- with .Values.vector.serviceMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + endpoints: + - port: metrics + interval: 30s + scrapeTimeout: 5s + selector: + matchLabels: + {{- include "seaweedfs.vector.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/k8s/charts/seaweedfs/tests/vector_bucket_test.go b/k8s/charts/seaweedfs/tests/vector_bucket_test.go new file mode 100644 index 0000000000..bec5487edf --- /dev/null +++ b/k8s/charts/seaweedfs/tests/vector_bucket_test.go @@ -0,0 +1,113 @@ +package tests + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestVectorBucketHook(t *testing.T) { + for _, auth := range []bool{false, true} { + t.Run(fmt.Sprint(auth), func(t *testing.T) { + output, documents := render(t, "s3.enabled=true", "vector.enabled=true", "vector.createBuckets[0]=embeddings", "s3.enableAuth="+fmt.Sprint(auth), "networkPolicy.enabled=true") + var found bool + for _, document := range documents { + metadata := document["metadata"].(map[string]interface{}) + if document["kind"] == "Job" && strings.HasSuffix(metadata["name"].(string), "-vector-bucket-hook") { + found = true + if metadata["annotations"].(map[string]interface{})["helm.sh/hook"] != "post-install,post-upgrade" { + t.Fatal("missing upgrade hook") + } + } + } + if !found || !strings.Contains(output, "s3vectors get-vector-bucket") || !strings.Contains(output, "s3vectors create-vector-bucket") { + t.Fatal("missing vector bucket provisioning") + } + if auth && !strings.Contains(output, "key: \"admin_access_key_id\"") { + t.Fatal("missing generated admin credential reference") + } + }) + } +} + +func TestVectorBucketHookRetriesRacesAndRefusesConflicts(t *testing.T) { + _, documents := render(t, "s3.enabled=true", "vector.enabled=true", "vector.createBuckets[0]=embeddings") + var script string + for _, document := range documents { + if document["kind"] != "Job" { + continue + } + metadata := document["metadata"].(map[string]interface{}) + if strings.HasSuffix(metadata["name"].(string), "-vector-bucket-hook") { + pod := document["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{}) + container := pod["containers"].([]interface{})[0].(map[string]interface{}) + script = container["args"].([]interface{})[0].(string) + } + } + if script == "" { + t.Fatal("missing hook script") + } + for _, mode := range []string{"missing", "existing", "race", "iceberg", "denied"} { + t.Run(mode, func(t *testing.T) { + dir := t.TempDir() + stub := `#!/bin/sh +shift 2 +operation=$2 +case "$operation" in +get-vector-bucket) + if [ "$BUCKET_MODE" = denied ]; then echo AccessDenied >&2; exit 1; fi + if [ "$BUCKET_MODE" = existing ] || { [ -f "$BUCKET_STATE" ] && [ "$BUCKET_MODE" != iceberg ]; }; then exit 0; fi + echo NotFoundException >&2; exit 1;; +create-vector-bucket) + touch "$BUCKET_STATE" + if [ "$BUCKET_MODE" = race ] || [ "$BUCKET_MODE" = iceberg ]; then exit 1; fi;; +esac +` + if err := os.WriteFile(filepath.Join(dir, "aws"), []byte(stub), 0755); err != nil { + t.Fatal(err) + } + command := exec.Command("sh", "-ec", script) + command.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "S3_ENDPOINT=http://unused", "BUCKET_MODE="+mode, "BUCKET_STATE="+filepath.Join(dir, "state")) + output, err := command.CombinedOutput() + wantFailure := mode == "iceberg" || mode == "denied" + if (err != nil) != wantFailure { + t.Fatalf("mode %s: %v %s", mode, err, output) + } + }) + } +} + +func TestVectorBucketHookRejectsShellNames(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + command := exec.Command("helm", "template", "test", "..", "--set", "s3.enabled=true", "--set", "vector.enabled=true", "--set", "vector.createBuckets[0]=bucket;touch /tmp/invalid") + output, err := command.CombinedOutput() + if err == nil || !strings.Contains(string(output), "vector.createBuckets names") { + t.Fatalf("invalid bucket accepted: %v %s", err, output) + } +} + +func TestVectorBucketHookGatewayModes(t *testing.T) { + for _, mode := range []string{"s3", "filer", "allInOne"} { + settings := []string{"vector.enabled=true", "vector.createBuckets[0]=embeddings"} + component := "s3" + switch mode { + case "s3": + settings = append(settings, "s3.enabled=true", "s3.enableAuth=true") + case "filer": + settings = append(settings, "s3.enabled=false", "filer.s3.enabled=true", "filer.s3.enableAuth=true") + component = "filer" + case "allInOne": + settings = append(settings, "s3.enabled=false", "allInOne.enabled=true", "allInOne.s3.enabled=true", "allInOne.s3.enableAuth=true") + component = "all-in-one" + } + output, _ := render(t, settings...) + if !strings.Contains(output, "http://vector-test-seaweedfs-"+component+".default:8333") || !strings.Contains(output, "key: \"admin_access_key_id\"") { + t.Fatalf("missing endpoint or auth for %s", mode) + } + } +} diff --git a/k8s/charts/seaweedfs/tests/vector_test.go b/k8s/charts/seaweedfs/tests/vector_test.go new file mode 100644 index 0000000000..d0d73546f6 --- /dev/null +++ b/k8s/charts/seaweedfs/tests/vector_test.go @@ -0,0 +1,148 @@ +package tests + +import ( + "fmt" + "io" + "os" + "os/exec" + "strings" + "testing" + + "go.yaml.in/yaml/v3" +) + +func render(t *testing.T, settings ...string) (string, []map[string]interface{}) { + t.Helper() + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + args := []string{"template", "vector-test", ".."} + for _, setting := range settings { + args = append(args, "--set", setting) + } + output, err := exec.Command("helm", args...).CombinedOutput() + if err != nil { + t.Fatalf("helm: %v\n%s", err, output) + } + decoder := yaml.NewDecoder(strings.NewReader(string(output))) + var documents []map[string]interface{} + for { + var document map[string]interface{} + if err := decoder.Decode(&document); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if document != nil { + documents = append(documents, document) + } + } + return string(output), documents +} + +func TestVectorDeployment(t *testing.T) { + _, defaults := render(t) + for _, document := range defaults { + if strings.HasSuffix(fmt.Sprint(document["metadata"].(map[string]interface{})["name"]), "-vector") { + t.Fatal("vector resource enabled by default") + } + } + for _, secure := range []bool{false, true} { + t.Run(fmt.Sprintf("security-%v", secure), func(t *testing.T) { + output, documents := render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test", "vector.port=8098", "global.seaweedfs.enableSecurity="+fmt.Sprint(secure), "global.seaweedfs.monitoring.enabled=true", "networkPolicy.enabled=true") + for _, want := range []string{"--listen=\"$POD_IP:8098\"", "--metrics-port=9329", "path: /ready", "path: /health", "kind: ServiceMonitor"} { + if !strings.Contains(output, want) { + t.Fatalf("missing %s", want) + } + } + found := false + for _, document := range documents { + metadata := document["metadata"].(map[string]interface{}) + if document["kind"] == "Deployment" && strings.HasSuffix(metadata["name"].(string), "-vector") { + found = true + spec := document["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{}) + if spec["nodeSelector"].(map[string]interface{})["kubernetes.io/os"] != "linux" { + t.Fatal("vector engine must use Linux image platforms") + } + } + } + if !found { + t.Fatal("no vector deployment") + } + if secure && (!strings.Contains(output, "[grpc.vector]") || !strings.Contains(output, "vector-cert")) { + t.Fatal("missing vector TLS configuration") + } + }) + } +} + +func TestVectorExistingImagesAndExternalCertificates(t *testing.T) { + output, _ := render(t, "vector.enabled=true") + start := strings.Index(output, "# Source: seaweedfs/templates/vector/vector-deployment.yaml") + deployment := strings.Split(output[start:], "\n---")[0] + if strings.Contains(deployment, "--metrics-port") || !strings.Contains(deployment, "tcpSocket:") { + t.Fatal("default probes must support existing vector images") + } + output, _ = render(t, "vector.enabled=true", "global.seaweedfs.enableSecurity=true", "certificates.externalCertificates.enabled=true") + if strings.Contains(output, "kind: Certificate") || !strings.Contains(output, "mountPath: /usr/local/share/ca-certificates/vector") { + t.Fatal("external vector certificates are not mounted") + } +} + +func TestVectorIPv6Listen(t *testing.T) { + _, docs := render(t, "vector.enabled=true") + var args []interface{} + for _, doc := range docs { + if doc["kind"] != "Deployment" { + continue + } + pod := doc["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{}) + c := pod["containers"].([]interface{})[0].(map[string]interface{}) + if c["name"] == "vector" { + args = c["args"].([]interface{}) + } + } + for _, ip := range []string{"10.0.0.20", "fd00::20"} { + dir := t.TempDir() + script := strings.ReplaceAll(args[0].(string), "/usr/bin/weed-vector", "printf '%s\\n'") + cmd := exec.Command("sh", "-ec", script, "vector", "--masters=master") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "POD_IP="+ip) + out, err := cmd.CombinedOutput() + address := ip + if strings.Contains(ip, ":") { + address = "[" + ip + "]" + } + if err != nil || !strings.Contains(string(out), "--listen="+address+":8090") || !strings.Contains(string(out), "--masters=master") { + t.Fatalf("%s: %s %v", ip, out, err) + } + } +} +func TestVectorAllInOneMasters(t *testing.T) { + output, _ := render(t, "vector.enabled=true", "allInOne.enabled=true", "master.enabled=false") + if !strings.Contains(output, "--masters=vector-test-seaweedfs-all-in-one.default:9333") { + t.Fatal("vector cannot discover all-in-one master") + } + output, _ = render(t, "vector.enabled=true", "allInOne.enabled=true", "master.enabled=false", "vector.masters=external:9333") + if !strings.Contains(output, "--masters=external:9333") { + t.Fatal("explicit master override lost") + } +} + +func TestVectorNetworkPolicyOverrides(t *testing.T) { + output, _ := render(t, "vector.enabled=true", "s3.enabled=true", "vector.createBuckets[0]=embeddings", "networkPolicy.enabled=true", "networkPolicy.components.vector.extraIngress[0].ports[0].port=9329", "networkPolicy.egress.enabled=true", "networkPolicy.components.vector-bucket-hook.extraEgress[0].ports[0].port=443") + if !strings.Contains(output, "port: 9329") || !strings.Contains(output, "port: 443") { + t.Fatal("vector network overrides missing") + } +} + +func TestVectorMetricsImageCompatibility(t *testing.T) { + if _, err := exec.LookPath("helm"); err != nil { + t.Skip("helm is not installed") + } + out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput() + if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") { + t.Fatalf("incompatible image accepted: %v %s", err, out) + } + render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test") +} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 51df7ca9a8..0ba8072314 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -1733,6 +1733,60 @@ admin: serviceMonitor: annotations: {} +vector: + enabled: false + imageOverride: null + replicas: 1 + port: 8090 + metricsPort: 0 + masters: "" + readCacheMB: 512 + nprobes: null + extraArgs: [] + extraEnvironmentVars: {} + podLabels: {} + podAnnotations: {} + annotations: {} + nodeSelector: + kubernetes.io/os: linux + tolerations: [] + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: [amd64, arm64] + resources: {} + podSecurityContext: {} + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + serviceMonitor: + annotations: {} + createBuckets: [] + bucketHook: + podSecurityContext: + seccompProfile: + type: RuntimeDefault + containerSecurityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: [ALL] + image: amazon/aws-cli:2.39.6 + endpoint: "" + region: us-east-1 + credentialsSecret: "" + accessKeyKey: AWS_ACCESS_KEY_ID + secretKeyKey: AWS_SECRET_ACCESS_KEY + sessionTokenKey: AWS_SESSION_TOKEN + caBundleSecret: "" + caBundleKey: tls.crt + resources: {} + worker: enabled: false imageOverride: null