mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-10 05:48:19 +01:00
helm: manage vector engines and provision vector buckets (#12518)
* helm: deploy vector engines with TLS, probes, and monitoring * helm: provision vector buckets through the S3 Vectors API * helm: bracket IPv6 pod addresses for vector listeners * helm: discover vector engines through the all-in-one master * helm: accept vector component NetworkPolicy overrides * helm: bind vector metrics to the pod address on both IP families * helm: provision vector buckets through embedded and all-in-one gateways * docs: use the full Helm name for external vector certificates * helm: render vector instance labels as strings * helm: allow vector bucket hooks to use restricted security contexts * helm: honor vector security context enable switches * helm: prevent duplicate vector monitoring labels * helm: reject metrics flags with the default pre-metrics vector image * test: skip vector chart compatibility checks when Helm is unavailable
This commit is contained in:
Chris Lu authored and GitHub committed
1 parent
c1afb1b1cb
commit
5abb18e678
14 files changed
+826
-2
No files matched your search
@@ -37,6 +37,13 @@ jobs:
|
||||
with:
|
||||
version: v3.18.4
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
|
||||
- name: Test vector chart resources
|
||||
run: go test ./k8s/charts/seaweedfs/tests -count=1
|
||||
|
||||
- uses: actions/setup-python@v7
|
||||
with:
|
||||
python-version: '3.10'
|
||||
|
||||
@@ -3,4 +3,4 @@ description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.53"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.53.0
|
||||
version: 4.53.1
|
||||
@@ -687,3 +687,47 @@ generates a new UUID and the license stops matching. See
|
||||
[Master data](#master-data-hostpath-vs-a-claim). Check the binding with
|
||||
`kubectl exec <master-pod> -- curl -s localhost:9333/license/status`
|
||||
(`cluster_uuid` must equal `license_uuid`).
|
||||
|
||||
|
||||
## Vector buckets
|
||||
|
||||
The S3 gateway serves the S3 Vectors API on its existing endpoint. Enable a vector engine and Lance maintenance worker alongside the cluster:
|
||||
|
||||
```yaml
|
||||
s3:
|
||||
enabled: true
|
||||
vector:
|
||||
enabled: true
|
||||
replicas: 1
|
||||
worker:
|
||||
enabled: true
|
||||
admin:
|
||||
enabled: true
|
||||
secret:
|
||||
adminPassword: change-me
|
||||
global:
|
||||
seaweedfs:
|
||||
securityConfig:
|
||||
jwtSigning:
|
||||
filerWrite: true
|
||||
```
|
||||
|
||||
The engine registers its pod address with the masters and stores indexes through the filers and volume servers. Both Go and Rust volume servers use the same setup. The maintenance worker builds and updates ANN indexes; queries can scan data before indexing completes. Enabling filer write signing also requires clients writing directly to the filer HTTP API to supply a signed JWT.
|
||||
|
||||
The bundled engine images support Linux amd64 and arm64. Use `vector.imageOverride` to select a compatible engine build, `vector.masters` for an external cluster, and `vector.resources` to set memory and CPU limits. The cache defaults to 512 MiB and is configured with `vector.readCacheMB`.
|
||||
|
||||
`global.seaweedfs.enableSecurity` provisions and mounts the vector certificate and shared security configuration. With external certificates, provide the `<fullname>-vector-cert` secret (`my-seaweedfs-vector-cert` for release `my`). `fullname` follows `seaweedfs.fullname`, including `fullnameOverride`, `nameOverride`, and DNS-name truncation. Provide this secret alongside the existing CA and client secrets.
|
||||
|
||||
The default `vector.metricsPort: 0` and TCP probes support existing engine images. The default 4.53 image does not accept metrics flags; enabling metrics with it is rejected at render time. For engine builds with `/metrics`, `/health`, and `/ready`, set `vector.imageOverride` to that compatible image and `vector.metricsPort: 9329`; readiness then checks master registration. Enabling `global.seaweedfs.monitoring.enabled` also creates the vector ServiceMonitor. NetworkPolicy includes the configured gRPC and metrics ports.
|
||||
|
||||
|
||||
To provision vector buckets during installation and upgrades, set:
|
||||
|
||||
```yaml
|
||||
vector:
|
||||
enabled: true
|
||||
createBuckets:
|
||||
- embeddings
|
||||
```
|
||||
|
||||
The hook uses the AWS CLI's S3 Vectors API and works with existing SeaweedFS vector releases. It creates missing buckets, preserves existing vector buckets, and fails if the name belongs to an Iceberg bucket. It uses the chart's generated S3 admin credentials, including `s3.credentials.admin.existingSecret`. With a custom `s3.existingConfigSecret` or an external endpoint, set `vector.bucketHook.credentialsSecret` to a Secret containing `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` (and optionally `AWS_SESSION_TOKEN`). Key names are configurable. `vector.bucketHook.endpoint` overrides the internal S3 service URL; `caBundleSecret` and `caBundleKey` configure a private HTTPS CA. The hook image and resource limits are configurable under `vector.bucketHook`. For restricted Pod Security, select a non-root-compatible image and configure `vector.bucketHook.podSecurityContext` and `containerSecurityContext`, including `runAsNonRoot` and the image user ID.
|
||||
@@ -0,0 +1,50 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if and .Values.vector.enabled .Values.global.seaweedfs.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.seaweedfs.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-vector-cert
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: vector
|
||||
{{- with include "seaweedfs.commonLabels" $ }}{{ . | nindent 4 }}{{ end }}
|
||||
{{- if .Values.vector.annotations }}
|
||||
annotations:
|
||||
{{- toYaml .Values.vector.annotations | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
secretName: {{ include "seaweedfs.fullname" . }}-vector-cert
|
||||
issuerRef:
|
||||
name: {{ include "seaweedfs.fullname" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-vector'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}.svc'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-vector.{{ include "seaweedfs.namespace" . }}.svc.cluster.local'
|
||||
{{- if .Values.certificates.dnsNames }}
|
||||
{{- range .Values.certificates.dnsNames }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
|
||||
@@ -770,3 +770,15 @@ on install.
|
||||
{{- $context := merge (dict "Values" (omit .Values "commonLabels")) (omit . "Values") -}}
|
||||
{{- include (print .Template.BasePath "/master/master-configmap.yaml") $context | sha256sum -}}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "seaweedfs.vector.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: vector
|
||||
{{- end -}}
|
||||
{{- define "seaweedfs.vector.labels" -}}
|
||||
{{ include "seaweedfs.vector.selectorLabels" . }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
{{- with include "seaweedfs.commonLabels" . }}{{ . | nindent 0 }}{{ end }}
|
||||
{{- end -}}
|
||||
@@ -110,6 +110,18 @@
|
||||
{{- $targets = append $targets (dict "component" "worker" "ports" $ports) }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.vector.enabled }}
|
||||
{{- $ports := list .Values.vector.port }}
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
{{- $ports = append $ports .Values.vector.metricsPort }}
|
||||
{{- end }}
|
||||
{{- $targets = append $targets (dict "component" "vector" "ports" $ports) }}
|
||||
{{- end }}
|
||||
|
||||
{{- if and .Values.vector.enabled .Values.vector.createBuckets }}
|
||||
{{- $targets = append $targets (dict "component" "vector-bucket-hook" "ports" (list)) }}
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.cosi.enabled }}
|
||||
{{- /* The provisioner sidecar watches the COSI CRs, so it needs the API
|
||||
server; nothing needs to reach the driver itself. */}}
|
||||
@@ -143,7 +155,7 @@
|
||||
every component the chart can produce rather than the ones enabled here,
|
||||
so one values file can still carry overrides for a component this release
|
||||
leaves off. */}}
|
||||
{{- $known := list "master" "filer" "s3" "sftp" "admin" "worker" "volume" "bucket-hook" "volume-resize-hook" "objectstorage-provisioner" "seaweedfs-all-in-one" }}
|
||||
{{- $known := list "vector" "vector-bucket-hook" "master" "filer" "s3" "sftp" "admin" "worker" "volume" "bucket-hook" "volume-resize-hook" "objectstorage-provisioner" "seaweedfs-all-in-one" }}
|
||||
{{- range $vname, $_ := (.Values.volumes | default dict) }}
|
||||
{{- $known = append $known (trimSuffix "-" (printf "volume-%s" $vname)) }}
|
||||
{{- end }}
|
||||
|
||||
@@ -94,6 +94,10 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/admin/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/admin/tls.key"
|
||||
|
||||
[grpc.vector]
|
||||
cert = "/usr/local/share/ca-certificates/vector/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/vector/tls.key"
|
||||
|
||||
[grpc.worker]
|
||||
cert = "/usr/local/share/ca-certificates/worker/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/worker/tls.key"
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if and .Values.vector.enabled .Values.vector.createBuckets }}
|
||||
{{- $hook := .Values.vector.bucketHook }}
|
||||
{{- $component := "s3" }}
|
||||
{{- $port := .Values.s3.port }}
|
||||
{{- $enabled := .Values.s3.enabled }}
|
||||
{{- $auth := .Values.s3.enableAuth }}
|
||||
{{- $configSecret := .Values.s3.existingConfigSecret }}
|
||||
{{- if and .Values.allInOne.enabled .Values.allInOne.s3.enabled }}
|
||||
{{- $component = "all-in-one" }}
|
||||
{{- $port = .Values.allInOne.s3.port | default .Values.s3.port }}
|
||||
{{- $enabled = true }}
|
||||
{{- $auth = or .Values.allInOne.s3.enableAuth .Values.s3.enableAuth .Values.filer.s3.enableAuth }}
|
||||
{{- $configSecret = or .Values.allInOne.s3.existingConfigSecret .Values.s3.existingConfigSecret .Values.filer.s3.existingConfigSecret }}
|
||||
{{- else if and (not $enabled) .Values.filer.s3.enabled }}
|
||||
{{- $component = "filer" }}
|
||||
{{- $port = .Values.filer.s3.port }}
|
||||
{{- $enabled = true }}
|
||||
{{- $auth = .Values.filer.s3.enableAuth }}
|
||||
{{- $configSecret = .Values.filer.s3.existingConfigSecret }}
|
||||
{{- end }}
|
||||
{{- if and (not $enabled) (not $hook.endpoint) }}
|
||||
{{- fail "vector.createBuckets requires an enabled S3 gateway or vector.bucketHook.endpoint" }}
|
||||
{{- end }}
|
||||
{{- range .Values.vector.createBuckets }}
|
||||
{{- if not (kindIs "string" .) }}{{ fail "vector.createBuckets entries must be bucket names" }}{{ end }}
|
||||
{{- if not (and (ge (len .) 3) (le (len .) 63) (regexMatch "^[a-z0-9][a-z0-9-]*[a-z0-9]$" .)) }}
|
||||
{{- fail "vector.createBuckets names must be 3-63 lowercase letters, digits or hyphens" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- $credentials := .Values.s3.credentials | default dict }}
|
||||
{{- $admin := $credentials.admin | default dict }}
|
||||
{{- $secret := $hook.credentialsSecret }}
|
||||
{{- $accessKey := $hook.accessKeyKey }}
|
||||
{{- $secretKey := $hook.secretKeyKey }}
|
||||
{{- if and $auth (not $secret) }}
|
||||
{{- if $configSecret }}{{ fail "vector.bucketHook.credentialsSecret is required with s3.existingConfigSecret" }}{{ end }}
|
||||
{{- $secret = $admin.existingSecret | default (printf "%s-s3-secret" (include "seaweedfs.fullname" .)) }}
|
||||
{{- $accessKey = $admin.accessKeyKey | default "admin_access_key_id" }}
|
||||
{{- $secretKey = $admin.secretKeyKey | default "admin_secret_access_key" }}
|
||||
{{- end }}
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.fullname" . }}-vector-bucket-hook
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: vector-bucket-hook
|
||||
{{- with include "seaweedfs.commonLabels" . }}{{ . | nindent 4 }}{{ end }}
|
||||
annotations:
|
||||
helm.sh/hook: post-install,post-upgrade
|
||||
helm.sh/hook-weight: "-5"
|
||||
helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
activeDeadlineSeconds: 360
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ include "seaweedfs.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name | quote }}
|
||||
app.kubernetes.io/component: vector-bucket-hook
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
automountServiceAccountToken: false
|
||||
{{- with $hook.podSecurityContext }}
|
||||
securityContext:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.vector.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.vector.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
containers:
|
||||
- name: create-vector-buckets
|
||||
image: {{ $hook.image | quote }}
|
||||
command: ["/bin/sh", "-ec"]
|
||||
args:
|
||||
- |
|
||||
for bucket in {{ range .Values.vector.createBuckets }}{{ . | quote }} {{ end }}; do
|
||||
attempt=0
|
||||
while ! aws --endpoint-url "$S3_ENDPOINT" s3vectors get-vector-bucket --vector-bucket-name "$bucket" >/dev/null 2>/tmp/bucket-error; do
|
||||
if grep -q NotFound /tmp/bucket-error; then
|
||||
aws --endpoint-url "$S3_ENDPOINT" s3vectors create-vector-bucket --vector-bucket-name "$bucket" ||
|
||||
aws --endpoint-url "$S3_ENDPOINT" s3vectors get-vector-bucket --vector-bucket-name "$bucket"
|
||||
break
|
||||
fi
|
||||
if grep -q AccessDenied /tmp/bucket-error; then cat /tmp/bucket-error >&2; exit 1; fi
|
||||
attempt=$((attempt + 1))
|
||||
if [ "$attempt" -ge 60 ]; then cat /tmp/bucket-error >&2; exit 1; fi
|
||||
sleep 5
|
||||
done
|
||||
done
|
||||
env:
|
||||
- name: S3_ENDPOINT
|
||||
value: {{ $hook.endpoint | default (printf "http://%s.%s:%d" (include "seaweedfs.componentName" (list . $component)) (include "seaweedfs.namespace" .) (int $port)) | quote }}
|
||||
- name: AWS_DEFAULT_REGION
|
||||
value: {{ $hook.region | quote }}
|
||||
- name: AWS_EC2_METADATA_DISABLED
|
||||
value: "true"
|
||||
- name: AWS_PAGER
|
||||
value: ""
|
||||
- name: AWS_MAX_ATTEMPTS
|
||||
value: "2"
|
||||
- name: AWS_ACCESS_KEY_ID
|
||||
{{- if $secret }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $secret | quote }}
|
||||
key: {{ $accessKey | quote }}
|
||||
{{- else }}
|
||||
value: seaweedfs
|
||||
{{- end }}
|
||||
- name: AWS_SECRET_ACCESS_KEY
|
||||
{{- if $secret }}
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $secret | quote }}
|
||||
key: {{ $secretKey | quote }}
|
||||
{{- else }}
|
||||
value: seaweedfs
|
||||
{{- end }}
|
||||
{{- if $hook.credentialsSecret }}
|
||||
- name: AWS_SESSION_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: {{ $secret | quote }}
|
||||
key: {{ $hook.sessionTokenKey | quote }}
|
||||
optional: true
|
||||
{{- end }}
|
||||
{{- if $hook.caBundleSecret }}
|
||||
- name: AWS_CA_BUNDLE
|
||||
value: /etc/ssl/vector/ca.crt
|
||||
{{- end }}
|
||||
securityContext:
|
||||
{{- toYaml $hook.containerSecurityContext | nindent 12 }}
|
||||
{{- with $hook.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
{{- if $hook.caBundleSecret }}
|
||||
- name: ca
|
||||
mountPath: /etc/ssl/vector
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- if $hook.caBundleSecret }}
|
||||
- name: ca
|
||||
secret:
|
||||
secretName: {{ $hook.caBundleSecret | quote }}
|
||||
items:
|
||||
- key: {{ $hook.caBundleKey | quote }}
|
||||
path: ca.crt
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,166 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.vector.enabled }}
|
||||
{{- if and .Values.vector.metricsPort (not .Values.vector.imageOverride) (eq (toString (.Values.image.tag | default .Chart.AppVersion)) "4.53") }}
|
||||
{{- fail "vector.metricsPort requires vector.imageOverride selecting an engine with metrics support; the default 4.53 image does not support metrics" }}
|
||||
{{- end }}
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "vector") }}
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
{{- include "seaweedfs.vector.labels" . | nindent 4 }}
|
||||
{{- with .Values.vector.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
replicas: {{ .Values.vector.replicas }}
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "seaweedfs.vector.selectorLabels" . | nindent 6 }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "seaweedfs.vector.selectorLabels" . | nindent 8 }}
|
||||
{{- with .Values.podLabels }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with .Values.vector.podLabels }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
annotations:
|
||||
{{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }}
|
||||
{{- with .Values.vector.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.vector.affinity }}
|
||||
affinity:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.vector.tolerations }}
|
||||
tolerations:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- if and .Values.vector.podSecurityContext (or (not (hasKey .Values.vector.podSecurityContext "enabled")) .Values.vector.podSecurityContext.enabled) }}
|
||||
securityContext:
|
||||
{{- omit .Values.vector.podSecurityContext "enabled" | toYaml | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: vector
|
||||
image: {{ .Values.vector.imageOverride | default (include "seaweedfs.image" .) | quote }}
|
||||
imagePullPolicy: {{ .Values.global.seaweedfs.imagePullPolicy }}
|
||||
{{- $masters := include "seaweedfs.masterServerArg" . }}
|
||||
{{- if .Values.allInOne.enabled }}{{ $masters = include "seaweedfs.cluster.masterAddress" . }}{{ end }}
|
||||
command: ["/bin/sh", "-ec"]
|
||||
args:
|
||||
- 'case "$POD_IP" in *:*) POD_IP="[$POD_IP]";; esac; exec /usr/bin/weed-vector --listen="$POD_IP:{{ .Values.vector.port }}" "$@"'
|
||||
- vector
|
||||
- {{ printf "--masters=%s" (.Values.vector.masters | default $masters) | quote }}
|
||||
- {{ printf "--read-cache-mb=%d" (int .Values.vector.readCacheMB) | quote }}
|
||||
{{- if .Values.filer.filerGroup }}
|
||||
- {{ printf "--filer-group=%s" .Values.filer.filerGroup | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.vector.nprobes }}
|
||||
- {{ printf "--nprobes=%d" (int .Values.vector.nprobes) | quote }}
|
||||
{{- end }}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- --security-toml=/etc/seaweedfs/security.toml
|
||||
{{- end }}
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
- {{ printf "--metrics-port=%d" (int .Values.vector.metricsPort) | quote }}
|
||||
- --metrics-ip=$(POD_IP)
|
||||
{{- end }}
|
||||
{{- range .Values.vector.extraArgs }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
{{- $env := dict }}
|
||||
{{- include "seaweedfs.mergeExtraEnvironmentVars" (dict "global" .Values.global.seaweedfs "component" .Values.vector "target" $env) }}
|
||||
{{- range $key := keys $env | sortAlpha }}
|
||||
- name: {{ $key }}
|
||||
{{- $value := index $env $key }}
|
||||
{{- if kindIs "map" $value }}
|
||||
valueFrom:
|
||||
{{- toYaml $value | nindent 16 }}
|
||||
{{- else }}
|
||||
value: {{ tpl (toString $value) $ | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- name: grpc
|
||||
containerPort: {{ .Values.vector.port }}
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
- name: metrics
|
||||
containerPort: {{ .Values.vector.metricsPort }}
|
||||
{{- end }}
|
||||
startupProbe:
|
||||
tcpSocket:
|
||||
port: grpc
|
||||
periodSeconds: 5
|
||||
failureThreshold: 60
|
||||
readinessProbe:
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
httpGet:
|
||||
path: /ready
|
||||
port: metrics
|
||||
{{- else }}
|
||||
tcpSocket:
|
||||
port: grpc
|
||||
{{- end }}
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
httpGet:
|
||||
path: /health
|
||||
port: metrics
|
||||
{{- else }}
|
||||
tcpSocket:
|
||||
port: grpc
|
||||
{{- end }}
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
{{- if or (not (hasKey .Values.vector.containerSecurityContext "enabled")) .Values.vector.containerSecurityContext.enabled }}
|
||||
securityContext:
|
||||
{{- omit .Values.vector.containerSecurityContext "enabled" | toYaml | nindent 12 }}
|
||||
{{- end }}
|
||||
{{- with .Values.vector.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
volumeMounts:
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
mountPath: /etc/seaweedfs
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- range list "ca" "client" "vector" }}
|
||||
- name: {{ . }}-cert
|
||||
mountPath: /usr/local/share/ca-certificates/{{ . }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
{{- if include "seaweedfs.securityConfigEnabled" . }}
|
||||
- name: security-config
|
||||
configMap:
|
||||
name: {{ include "seaweedfs.fullname" . }}-security-config
|
||||
{{- end }}
|
||||
{{- if .Values.global.seaweedfs.enableSecurity }}
|
||||
{{- range list "ca" "client" "vector" }}
|
||||
- name: {{ . }}-cert
|
||||
secret:
|
||||
secretName: {{ include "seaweedfs.fullname" $ }}-{{ . }}-cert
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,22 @@
|
||||
{{- if .Values.vector.enabled }}
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "vector") }}
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
{{- include "seaweedfs.vector.labels" . | nindent 4 }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
selector:
|
||||
{{- include "seaweedfs.vector.selectorLabels" . | nindent 4 }}
|
||||
ports:
|
||||
- name: grpc
|
||||
port: {{ .Values.vector.port }}
|
||||
targetPort: grpc
|
||||
{{- if .Values.vector.metricsPort }}
|
||||
- name: metrics
|
||||
port: {{ .Values.vector.metricsPort }}
|
||||
targetPort: metrics
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,25 @@
|
||||
{{- if and .Values.vector.enabled .Values.vector.metricsPort .Values.global.seaweedfs.monitoring.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: ServiceMonitor
|
||||
metadata:
|
||||
name: {{ include "seaweedfs.componentName" (list . "vector") }}
|
||||
namespace: {{ include "seaweedfs.namespace" . | quote }}
|
||||
labels:
|
||||
{{- include "seaweedfs.vector.selectorLabels" . | nindent 4 }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
{{- with include "seaweedfs.commonLabels" (dict "context" $ "exclude" $.Values.global.seaweedfs.monitoring.additionalLabels) }}{{ . | nindent 4 }}{{ end }}
|
||||
{{- with .Values.global.seaweedfs.monitoring.additionalLabels }}{{ toYaml . | nindent 4 }}{{ end }}
|
||||
{{- with .Values.vector.serviceMonitor.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
endpoints:
|
||||
- port: metrics
|
||||
interval: 30s
|
||||
scrapeTimeout: 5s
|
||||
selector:
|
||||
matchLabels:
|
||||
{{- include "seaweedfs.vector.selectorLabels" . | nindent 6 }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,113 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestVectorBucketHook(t *testing.T) {
|
||||
for _, auth := range []bool{false, true} {
|
||||
t.Run(fmt.Sprint(auth), func(t *testing.T) {
|
||||
output, documents := render(t, "s3.enabled=true", "vector.enabled=true", "vector.createBuckets[0]=embeddings", "s3.enableAuth="+fmt.Sprint(auth), "networkPolicy.enabled=true")
|
||||
var found bool
|
||||
for _, document := range documents {
|
||||
metadata := document["metadata"].(map[string]interface{})
|
||||
if document["kind"] == "Job" && strings.HasSuffix(metadata["name"].(string), "-vector-bucket-hook") {
|
||||
found = true
|
||||
if metadata["annotations"].(map[string]interface{})["helm.sh/hook"] != "post-install,post-upgrade" {
|
||||
t.Fatal("missing upgrade hook")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found || !strings.Contains(output, "s3vectors get-vector-bucket") || !strings.Contains(output, "s3vectors create-vector-bucket") {
|
||||
t.Fatal("missing vector bucket provisioning")
|
||||
}
|
||||
if auth && !strings.Contains(output, "key: \"admin_access_key_id\"") {
|
||||
t.Fatal("missing generated admin credential reference")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorBucketHookRetriesRacesAndRefusesConflicts(t *testing.T) {
|
||||
_, documents := render(t, "s3.enabled=true", "vector.enabled=true", "vector.createBuckets[0]=embeddings")
|
||||
var script string
|
||||
for _, document := range documents {
|
||||
if document["kind"] != "Job" {
|
||||
continue
|
||||
}
|
||||
metadata := document["metadata"].(map[string]interface{})
|
||||
if strings.HasSuffix(metadata["name"].(string), "-vector-bucket-hook") {
|
||||
pod := document["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})
|
||||
container := pod["containers"].([]interface{})[0].(map[string]interface{})
|
||||
script = container["args"].([]interface{})[0].(string)
|
||||
}
|
||||
}
|
||||
if script == "" {
|
||||
t.Fatal("missing hook script")
|
||||
}
|
||||
for _, mode := range []string{"missing", "existing", "race", "iceberg", "denied"} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
stub := `#!/bin/sh
|
||||
shift 2
|
||||
operation=$2
|
||||
case "$operation" in
|
||||
get-vector-bucket)
|
||||
if [ "$BUCKET_MODE" = denied ]; then echo AccessDenied >&2; exit 1; fi
|
||||
if [ "$BUCKET_MODE" = existing ] || { [ -f "$BUCKET_STATE" ] && [ "$BUCKET_MODE" != iceberg ]; }; then exit 0; fi
|
||||
echo NotFoundException >&2; exit 1;;
|
||||
create-vector-bucket)
|
||||
touch "$BUCKET_STATE"
|
||||
if [ "$BUCKET_MODE" = race ] || [ "$BUCKET_MODE" = iceberg ]; then exit 1; fi;;
|
||||
esac
|
||||
`
|
||||
if err := os.WriteFile(filepath.Join(dir, "aws"), []byte(stub), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
command := exec.Command("sh", "-ec", script)
|
||||
command.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "S3_ENDPOINT=http://unused", "BUCKET_MODE="+mode, "BUCKET_STATE="+filepath.Join(dir, "state"))
|
||||
output, err := command.CombinedOutput()
|
||||
wantFailure := mode == "iceberg" || mode == "denied"
|
||||
if (err != nil) != wantFailure {
|
||||
t.Fatalf("mode %s: %v %s", mode, err, output)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorBucketHookRejectsShellNames(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
command := exec.Command("helm", "template", "test", "..", "--set", "s3.enabled=true", "--set", "vector.enabled=true", "--set", "vector.createBuckets[0]=bucket;touch /tmp/invalid")
|
||||
output, err := command.CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(output), "vector.createBuckets names") {
|
||||
t.Fatalf("invalid bucket accepted: %v %s", err, output)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorBucketHookGatewayModes(t *testing.T) {
|
||||
for _, mode := range []string{"s3", "filer", "allInOne"} {
|
||||
settings := []string{"vector.enabled=true", "vector.createBuckets[0]=embeddings"}
|
||||
component := "s3"
|
||||
switch mode {
|
||||
case "s3":
|
||||
settings = append(settings, "s3.enabled=true", "s3.enableAuth=true")
|
||||
case "filer":
|
||||
settings = append(settings, "s3.enabled=false", "filer.s3.enabled=true", "filer.s3.enableAuth=true")
|
||||
component = "filer"
|
||||
case "allInOne":
|
||||
settings = append(settings, "s3.enabled=false", "allInOne.enabled=true", "allInOne.s3.enabled=true", "allInOne.s3.enableAuth=true")
|
||||
component = "all-in-one"
|
||||
}
|
||||
output, _ := render(t, settings...)
|
||||
if !strings.Contains(output, "http://vector-test-seaweedfs-"+component+".default:8333") || !strings.Contains(output, "key: \"admin_access_key_id\"") {
|
||||
t.Fatalf("missing endpoint or auth for %s", mode)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
func render(t *testing.T, settings ...string) (string, []map[string]interface{}) {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
args := []string{"template", "vector-test", ".."}
|
||||
for _, setting := range settings {
|
||||
args = append(args, "--set", setting)
|
||||
}
|
||||
output, err := exec.Command("helm", args...).CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("helm: %v\n%s", err, output)
|
||||
}
|
||||
decoder := yaml.NewDecoder(strings.NewReader(string(output)))
|
||||
var documents []map[string]interface{}
|
||||
for {
|
||||
var document map[string]interface{}
|
||||
if err := decoder.Decode(&document); err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if document != nil {
|
||||
documents = append(documents, document)
|
||||
}
|
||||
}
|
||||
return string(output), documents
|
||||
}
|
||||
|
||||
func TestVectorDeployment(t *testing.T) {
|
||||
_, defaults := render(t)
|
||||
for _, document := range defaults {
|
||||
if strings.HasSuffix(fmt.Sprint(document["metadata"].(map[string]interface{})["name"]), "-vector") {
|
||||
t.Fatal("vector resource enabled by default")
|
||||
}
|
||||
}
|
||||
for _, secure := range []bool{false, true} {
|
||||
t.Run(fmt.Sprintf("security-%v", secure), func(t *testing.T) {
|
||||
output, documents := render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test", "vector.port=8098", "global.seaweedfs.enableSecurity="+fmt.Sprint(secure), "global.seaweedfs.monitoring.enabled=true", "networkPolicy.enabled=true")
|
||||
for _, want := range []string{"--listen=\"$POD_IP:8098\"", "--metrics-port=9329", "path: /ready", "path: /health", "kind: ServiceMonitor"} {
|
||||
if !strings.Contains(output, want) {
|
||||
t.Fatalf("missing %s", want)
|
||||
}
|
||||
}
|
||||
found := false
|
||||
for _, document := range documents {
|
||||
metadata := document["metadata"].(map[string]interface{})
|
||||
if document["kind"] == "Deployment" && strings.HasSuffix(metadata["name"].(string), "-vector") {
|
||||
found = true
|
||||
spec := document["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})
|
||||
if spec["nodeSelector"].(map[string]interface{})["kubernetes.io/os"] != "linux" {
|
||||
t.Fatal("vector engine must use Linux image platforms")
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("no vector deployment")
|
||||
}
|
||||
if secure && (!strings.Contains(output, "[grpc.vector]") || !strings.Contains(output, "vector-cert")) {
|
||||
t.Fatal("missing vector TLS configuration")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorExistingImagesAndExternalCertificates(t *testing.T) {
|
||||
output, _ := render(t, "vector.enabled=true")
|
||||
start := strings.Index(output, "# Source: seaweedfs/templates/vector/vector-deployment.yaml")
|
||||
deployment := strings.Split(output[start:], "\n---")[0]
|
||||
if strings.Contains(deployment, "--metrics-port") || !strings.Contains(deployment, "tcpSocket:") {
|
||||
t.Fatal("default probes must support existing vector images")
|
||||
}
|
||||
output, _ = render(t, "vector.enabled=true", "global.seaweedfs.enableSecurity=true", "certificates.externalCertificates.enabled=true")
|
||||
if strings.Contains(output, "kind: Certificate") || !strings.Contains(output, "mountPath: /usr/local/share/ca-certificates/vector") {
|
||||
t.Fatal("external vector certificates are not mounted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorIPv6Listen(t *testing.T) {
|
||||
_, docs := render(t, "vector.enabled=true")
|
||||
var args []interface{}
|
||||
for _, doc := range docs {
|
||||
if doc["kind"] != "Deployment" {
|
||||
continue
|
||||
}
|
||||
pod := doc["spec"].(map[string]interface{})["template"].(map[string]interface{})["spec"].(map[string]interface{})
|
||||
c := pod["containers"].([]interface{})[0].(map[string]interface{})
|
||||
if c["name"] == "vector" {
|
||||
args = c["args"].([]interface{})
|
||||
}
|
||||
}
|
||||
for _, ip := range []string{"10.0.0.20", "fd00::20"} {
|
||||
dir := t.TempDir()
|
||||
script := strings.ReplaceAll(args[0].(string), "/usr/bin/weed-vector", "printf '%s\\n'")
|
||||
cmd := exec.Command("sh", "-ec", script, "vector", "--masters=master")
|
||||
cmd.Dir = dir
|
||||
cmd.Env = append(os.Environ(), "POD_IP="+ip)
|
||||
out, err := cmd.CombinedOutput()
|
||||
address := ip
|
||||
if strings.Contains(ip, ":") {
|
||||
address = "[" + ip + "]"
|
||||
}
|
||||
if err != nil || !strings.Contains(string(out), "--listen="+address+":8090") || !strings.Contains(string(out), "--masters=master") {
|
||||
t.Fatalf("%s: %s %v", ip, out, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestVectorAllInOneMasters(t *testing.T) {
|
||||
output, _ := render(t, "vector.enabled=true", "allInOne.enabled=true", "master.enabled=false")
|
||||
if !strings.Contains(output, "--masters=vector-test-seaweedfs-all-in-one.default:9333") {
|
||||
t.Fatal("vector cannot discover all-in-one master")
|
||||
}
|
||||
output, _ = render(t, "vector.enabled=true", "allInOne.enabled=true", "master.enabled=false", "vector.masters=external:9333")
|
||||
if !strings.Contains(output, "--masters=external:9333") {
|
||||
t.Fatal("explicit master override lost")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorNetworkPolicyOverrides(t *testing.T) {
|
||||
output, _ := render(t, "vector.enabled=true", "s3.enabled=true", "vector.createBuckets[0]=embeddings", "networkPolicy.enabled=true", "networkPolicy.components.vector.extraIngress[0].ports[0].port=9329", "networkPolicy.egress.enabled=true", "networkPolicy.components.vector-bucket-hook.extraEgress[0].ports[0].port=443")
|
||||
if !strings.Contains(output, "port: 9329") || !strings.Contains(output, "port: 443") {
|
||||
t.Fatal("vector network overrides missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVectorMetricsImageCompatibility(t *testing.T) {
|
||||
if _, err := exec.LookPath("helm"); err != nil {
|
||||
t.Skip("helm is not installed")
|
||||
}
|
||||
out, err := exec.Command("helm", "template", "vector-test", "..", "--set", "vector.enabled=true", "--set", "vector.metricsPort=9329").CombinedOutput()
|
||||
if err == nil || !strings.Contains(string(out), "default 4.53 image does not support metrics") {
|
||||
t.Fatalf("incompatible image accepted: %v %s", err, out)
|
||||
}
|
||||
render(t, "vector.enabled=true", "vector.metricsPort=9329", "vector.imageOverride=example/metrics-engine:test")
|
||||
}
|
||||
@@ -1733,6 +1733,60 @@ admin:
|
||||
serviceMonitor:
|
||||
annotations: {}
|
||||
|
||||
vector:
|
||||
enabled: false
|
||||
imageOverride: null
|
||||
replicas: 1
|
||||
port: 8090
|
||||
metricsPort: 0
|
||||
masters: ""
|
||||
readCacheMB: 512
|
||||
nprobes: null
|
||||
extraArgs: []
|
||||
extraEnvironmentVars: {}
|
||||
podLabels: {}
|
||||
podAnnotations: {}
|
||||
annotations: {}
|
||||
nodeSelector:
|
||||
kubernetes.io/os: linux
|
||||
tolerations: []
|
||||
affinity:
|
||||
nodeAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
nodeSelectorTerms:
|
||||
- matchExpressions:
|
||||
- key: kubernetes.io/arch
|
||||
operator: In
|
||||
values: [amd64, arm64]
|
||||
resources: {}
|
||||
podSecurityContext: {}
|
||||
containerSecurityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
serviceMonitor:
|
||||
annotations: {}
|
||||
createBuckets: []
|
||||
bucketHook:
|
||||
podSecurityContext:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containerSecurityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
image: amazon/aws-cli:2.39.6
|
||||
endpoint: ""
|
||||
region: us-east-1
|
||||
credentialsSecret: ""
|
||||
accessKeyKey: AWS_ACCESS_KEY_ID
|
||||
secretKeyKey: AWS_SECRET_ACCESS_KEY
|
||||
sessionTokenKey: AWS_SESSION_TOKEN
|
||||
caBundleSecret: ""
|
||||
caBundleKey: tls.crt
|
||||
resources: {}
|
||||
|
||||
worker:
|
||||
enabled: false
|
||||
imageOverride: null
|
||||
|
||||
Reference in new issue
Block a user