mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 19:33:41 +01:00
Compare commits
14
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c7287f8f54 | ||
|
|
e00698a327 | ||
|
|
63a575f28f | ||
|
|
ed67f9930c | ||
|
|
dd4d06e40f | ||
|
|
64d5fb5613 | ||
|
|
e15aa250e9 | ||
|
|
c908497d56 | ||
|
|
9bc1a50b37 | ||
|
|
dc2d13a1ee | ||
|
|
af35fb158c | ||
|
|
080af98b86 | ||
|
|
4c92a00357 | ||
|
|
1d7125abf2 |
No files matched your search
@@ -259,6 +259,40 @@ jobs:
|
||||
grep -q "security-config" /tmp/security.yaml
|
||||
echo "Security configuration renders correctly"
|
||||
|
||||
echo "=== Testing volume server HTTPS ==="
|
||||
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/security.yaml | grep -q 'enabled = false'
|
||||
helm template test $CHART_DIR \
|
||||
--set global.seaweedfs.enableSecurity=true \
|
||||
--set global.seaweedfs.volumeHttps.enabled=true \
|
||||
--set volumes.hot.replicas=1 \
|
||||
> /tmp/volume-https.yaml
|
||||
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml > /tmp/volume-https-client.txt
|
||||
grep -q 'enabled = true' /tmp/volume-https-client.txt
|
||||
grep -q 'cert = "/usr/local/share/ca-certificates/client/tls.crt"' /tmp/volume-https-client.txt
|
||||
grep -q 'ca = "/usr/local/share/ca-certificates/ca/tls.crt"' /tmp/volume-https-client.txt
|
||||
awk '/^ *\[https.volume\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml \
|
||||
| grep -q 'cert = "/usr/local/share/ca-certificates/volume/tls.crt"'
|
||||
grep -q "'\*.test-seaweedfs-volume.default'" /tmp/volume-https.yaml
|
||||
grep -q "'\*.test-seaweedfs-volume-hot.default'" /tmp/volume-https.yaml
|
||||
test "$(grep -c 'scheme: HTTPS' /tmp/volume-https.yaml)" -eq 4
|
||||
test "$(grep -c 'checksum/volume-https:' /tmp/volume-https.yaml)" -eq 4
|
||||
if grep -q 'checksum/volume-https' /tmp/security.yaml; then
|
||||
echo "FAIL: the volume HTTPS rollout annotation should only render while the option is on"
|
||||
exit 1
|
||||
fi
|
||||
if helm template test $CHART_DIR --set global.seaweedfs.volumeHttps.enabled=true > /dev/null 2>/tmp/volume-https-nosec.err; then
|
||||
echo "FAIL: volumeHttps without enableSecurity should fail to render"
|
||||
exit 1
|
||||
fi
|
||||
grep -q "requires global.seaweedfs.enableSecurity" /tmp/volume-https-nosec.err
|
||||
if helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true \
|
||||
--set global.seaweedfs.volumeHttps.enabled=true --set allInOne.enabled=true > /dev/null 2>/tmp/volume-https-aio.err; then
|
||||
echo "FAIL: volumeHttps with allInOne should fail to render"
|
||||
exit 1
|
||||
fi
|
||||
grep -q "not supported with allInOne" /tmp/volume-https-aio.err
|
||||
echo "Volume server HTTPS renders [https.volume], [https.client], certificate names, HTTPS probes and rollout annotations"
|
||||
|
||||
echo "=== Testing secure bucket-creation hook certificate mounts ==="
|
||||
helm template test $CHART_DIR \
|
||||
--show-only templates/shared/post-install-bucket-hook.yaml \
|
||||
|
||||
@@ -612,6 +612,17 @@ Two things worth knowing before you turn this on:
|
||||
|
||||
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
|
||||
|
||||
## Volume server HTTPS
|
||||
|
||||
With `global.seaweedfs.enableSecurity`, gRPC between components uses mutual TLS, but the volume servers' HTTP port, which carries the file data, stays plaintext. Set `global.seaweedfs.volumeHttps.enabled: true` to serve it over TLS:
|
||||
|
||||
- `[https.volume]` points at the volume certificate, which then also names every volume group's service (`*.<fullname>-volume.<namespace>`), since volume servers register as `<pod>.<fullname>-volume.<namespace>`. With `certificates.externalCertificates`, the provided `<fullname>-volume-cert` needs those names.
|
||||
- `[https.client]` is enabled with the client certificate and the chart CA, so every chart pod that talks to the volume servers does it over HTTPS.
|
||||
- The volume probes use HTTPS.
|
||||
- Every pod that reads `security.toml` gets a `checksum/volume-https` annotation, so switching the option on or off restarts them together.
|
||||
|
||||
The volume servers do not require client certificates, because kubelet probes cannot present one; enable JWT signing (`securityConfig.jwtSigning`) to authorize volume reads and writes. `[https.client]` also sends the admin UI's HTTP lookups on masters and S3 over HTTPS, which their plain listeners refuse, so the UI can miss volume public URLs and refuse SSE-S3 vector buckets. A volume Ingress or HTTPRoute needs its controller's HTTPS-backend setting. `weed mount`, the CSI driver and other clients outside the chart need their own `[https.client]` with the chart CA. The option is not supported with `allInOne`, whose volume server registers by pod IP.
|
||||
|
||||
## Common labels
|
||||
|
||||
`commonLabels` adds labels to the metadata of every object the chart creates. It may be a map or a template string rendering to one. The chart's own labels cannot be overridden, and pods (see `podLabels`) and `volumeClaimTemplates` are left unchanged.
|
||||
|
||||
@@ -56,6 +56,7 @@ spec:
|
||||
{{- with .Values.admin.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.admin.restartPolicy }}
|
||||
{{- if .Values.admin.affinity }}
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
{{- include "seaweedfs.compat" . -}}
|
||||
{{- if .Values.allInOne.enabled }}
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
{{- fail "global.seaweedfs.volumeHttps is not supported with allInOne: its volume server registers by pod IP, which the volume certificate cannot name" }}
|
||||
{{- end }}
|
||||
{{- $s3PreStopSeconds := 0 }}
|
||||
{{- if .Values.allInOne.s3.enabled }}
|
||||
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
|
||||
|
||||
@@ -38,6 +38,14 @@ spec:
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc'
|
||||
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc.cluster.local'
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
{{- range $vname, $_ := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
|
||||
{{- $volumeService := printf "%s.%s" (include "seaweedfs.componentName" (list $ (trimSuffix "-" (printf "volume-%s" $vname)))) (include "seaweedfs.namespace" $) }}
|
||||
- '*.{{ $volumeService }}'
|
||||
- '*.{{ $volumeService }}.svc'
|
||||
- '*.{{ $volumeService }}.svc.cluster.local'
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.certificates.ipAddresses }}
|
||||
ipAddresses:
|
||||
{{- range .Values.certificates.ipAddresses }}
|
||||
|
||||
@@ -39,6 +39,7 @@ spec:
|
||||
{{- with .Values.cosi.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.cosi.restartPolicy }}
|
||||
{{- if .Values.cosi.affinity }}
|
||||
|
||||
@@ -57,6 +57,7 @@ spec:
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.filer.restartPolicy }}
|
||||
{{- if .Values.filer.affinity }}
|
||||
|
||||
@@ -48,6 +48,7 @@ spec:
|
||||
{{- $_ := set $podAnnotations "checksum/master-config" (include "seaweedfs.master.configChecksum" .) }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.master.restartPolicy }}
|
||||
{{- if .Values.master.affinity }}
|
||||
|
||||
@@ -54,6 +54,7 @@ spec:
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{- toYaml $podAnnotations | nindent 8 }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.s3.restartPolicy }}
|
||||
{{- if .Values.s3.affinity }}
|
||||
|
||||
@@ -43,6 +43,7 @@ spec:
|
||||
{{- with .Values.sftp.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.sftp.restartPolicy }}
|
||||
{{- if .Values.sftp.affinity }}
|
||||
|
||||
@@ -491,6 +491,25 @@ true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* True when the volume servers serve HTTPS. Their certificates come from
|
||||
enableSecurity, so the option fails without it. */}}
|
||||
{{- define "seaweedfs.volumeHttps" -}}
|
||||
{{- if ((.Values.global.seaweedfs).volumeHttps).enabled -}}
|
||||
{{- if not .Values.global.seaweedfs.enableSecurity -}}
|
||||
{{- fail "global.seaweedfs.volumeHttps.enabled requires global.seaweedfs.enableSecurity, which provides the certificates" -}}
|
||||
{{- end -}}
|
||||
true
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Pod annotation that restarts the readers of security.toml when volume
|
||||
HTTPS is switched on or off. */}}
|
||||
{{- define "seaweedfs.volumeHttpsAnnotation" -}}
|
||||
{{- with include "seaweedfs.volumeHttps" . -}}
|
||||
checksum/volume-https: {{ sha256sum . | quote }}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Read a JWT key from the chart's existing security.toml without fromToml
|
||||
(which requires Helm >=3.17). Args: (list "<section>" $raw).
|
||||
Return the single-line TOML string token, including its quotes, so escapes
|
||||
|
||||
@@ -121,13 +121,25 @@ data:
|
||||
cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
|
||||
# volume server https options
|
||||
# Note: work in progress!
|
||||
# this does not work with other clients, e.g., "weed filer|mount" etc, yet.
|
||||
{{- if include "seaweedfs.volumeHttps" . }}
|
||||
|
||||
[https.client]
|
||||
enabled = true
|
||||
cert = "/usr/local/share/ca-certificates/client/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/client/tls.key"
|
||||
ca = "/usr/local/share/ca-certificates/ca/tls.crt"
|
||||
|
||||
[https.volume]
|
||||
cert = "/usr/local/share/ca-certificates/volume/tls.crt"
|
||||
key = "/usr/local/share/ca-certificates/volume/tls.key"
|
||||
{{- else }}
|
||||
|
||||
# volume server https: set global.seaweedfs.volumeHttps.enabled
|
||||
[https.client]
|
||||
enabled = false
|
||||
[https.volume]
|
||||
cert = ""
|
||||
key = ""
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -25,6 +25,7 @@ spec:
|
||||
annotations:
|
||||
{{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
|
||||
automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }}
|
||||
|
||||
@@ -55,6 +55,7 @@ spec:
|
||||
{{- with $volume.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
{{- if $volume.affinity }}
|
||||
affinity:
|
||||
@@ -281,7 +282,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ $volume.readinessProbe.httpGet.path }}
|
||||
port: {{ $volume.port }}
|
||||
scheme: {{ $volume.readinessProbe.httpGet.scheme }}
|
||||
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.readinessProbe.httpGet.scheme }}{{ end }}
|
||||
initialDelaySeconds: {{ $volume.readinessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ $volume.readinessProbe.periodSeconds }}
|
||||
successThreshold: {{ $volume.readinessProbe.successThreshold }}
|
||||
@@ -293,7 +294,7 @@ spec:
|
||||
httpGet:
|
||||
path: {{ $volume.livenessProbe.httpGet.path }}
|
||||
port: {{ $volume.port }}
|
||||
scheme: {{ $volume.livenessProbe.httpGet.scheme }}
|
||||
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.livenessProbe.httpGet.scheme }}{{ end }}
|
||||
initialDelaySeconds: {{ $volume.livenessProbe.initialDelaySeconds }}
|
||||
periodSeconds: {{ $volume.livenessProbe.periodSeconds }}
|
||||
successThreshold: {{ $volume.livenessProbe.successThreshold }}
|
||||
|
||||
@@ -46,6 +46,7 @@ spec:
|
||||
{{- with .Values.worker.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.worker.restartPolicy }}
|
||||
{{- if .Values.worker.affinity }}
|
||||
|
||||
@@ -45,6 +45,18 @@ global:
|
||||
volumeRead: 0
|
||||
filerWrite: 0
|
||||
filerRead: 0
|
||||
# Serve the volume servers' HTTP port over TLS with the chart's volume
|
||||
# certificate, and make every chart component reach them over HTTPS
|
||||
# ([https.volume] and [https.client] in security.toml). Requires
|
||||
# enableSecurity and is not supported with allInOne. Clients outside the
|
||||
# chart (weed mount, the CSI driver) need their own [https.client].
|
||||
# The volume servers do not ask for client certificates, because kubelet
|
||||
# probes cannot present one; JWT signing keeps authorizing volume access.
|
||||
# [https.client] also sends the admin UI's HTTP lookups on masters and
|
||||
# S3 over HTTPS, which their plain listeners refuse, so the UI can miss
|
||||
# volume public URLs and refuse SSE-S3 vector buckets.
|
||||
volumeHttps:
|
||||
enabled: false
|
||||
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
|
||||
serviceAccountName: "seaweedfs"
|
||||
serviceAccountAnnotations: {}
|
||||
|
||||
@@ -45,6 +45,9 @@ const (
|
||||
maxEstimatedRuntimeCap = 8 * time.Hour
|
||||
// How long started jobs may drain past the run window close.
|
||||
scheduledExecutionDrainGrace = 30 * time.Minute
|
||||
// A run whose jobs were given up for want of an executor detects again
|
||||
// this soon, instead of a whole detection interval later.
|
||||
capacityGiveUpRedetectDelay = 10 * time.Minute
|
||||
)
|
||||
|
||||
type schedulerPolicy struct {
|
||||
@@ -1033,8 +1036,9 @@ func (r *Plugin) dispatchScheduledProposals(
|
||||
// clears on a schedule the lane has no view of, and a lane that serialises
|
||||
// its job types would idle every other type in it for the whole runtime
|
||||
// budget; those jobs are given up now, with the reason, and re-proposed by
|
||||
// the next detection. A missing worker, or a free one that the queue's pin
|
||||
// keeps off a busy home, is waited for only as long as the orphan grace.
|
||||
// a detection brought forward to capacityGiveUpRedetectDelay. A missing
|
||||
// worker, or a free one that the queue's pin keeps off a busy home, is
|
||||
// waited for only as long as the orphan grace.
|
||||
func (r *Plugin) executorCapacityOutlook(jobType string, policy schedulerPolicy, blockedFor time.Duration) (bool, string) {
|
||||
executors, err := r.registry.ListExecutors(jobType)
|
||||
if err != nil || len(executors) == 0 {
|
||||
@@ -1093,9 +1097,22 @@ func (r *Plugin) giveUpQueuedJobs(queues *workerJobQueues, jobType, reason strin
|
||||
for _, item := range items {
|
||||
r.cancelQueuedJob(item.job, cause)
|
||||
}
|
||||
r.bringDetectionForward(jobType, capacityGiveUpRedetectDelay)
|
||||
return len(items)
|
||||
}
|
||||
|
||||
// bringDetectionForward schedules jobType's next detection within delay,
|
||||
// leaving an earlier one alone. A daily job given up because its worker
|
||||
// was briefly full would otherwise wait a day for its next chance.
|
||||
func (r *Plugin) bringDetectionForward(jobType string, delay time.Duration) {
|
||||
at := time.Now().UTC().Add(delay)
|
||||
r.schedulerMu.Lock()
|
||||
if next, ok := r.nextDetectionAt[jobType]; !ok || next.After(at) {
|
||||
r.nextDetectionAt[jobType] = at
|
||||
}
|
||||
r.schedulerMu.Unlock()
|
||||
}
|
||||
|
||||
func (r *Plugin) reserveScheduledExecutor(
|
||||
ctx context.Context,
|
||||
jobType string,
|
||||
|
||||
@@ -1759,3 +1759,80 @@ func TestCapacityReasonNamesUnitemizedSlots(t *testing.T) {
|
||||
t.Fatalf("outlook = (%v, %q), want (false, %q)", wait, reason, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A job type given up for capacity detects again soon: a daily job whose
|
||||
// worker was briefly full must not wait a whole interval for its next run.
|
||||
func TestCapacityGiveUpBringsNextDetectionForward(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
const workerID = "worker-a"
|
||||
pluginSvc.registry.UpsertFromHello(&plugin_pb.WorkerHello{
|
||||
WorkerId: workerID,
|
||||
Capabilities: []*plugin_pb.JobTypeCapability{
|
||||
{JobType: "admin_script", CanExecute: true, MaxExecutionConcurrency: 1},
|
||||
},
|
||||
})
|
||||
pluginSvc.registry.UpdateHeartbeat(workerID, &plugin_pb.WorkerHeartbeat{
|
||||
WorkerId: workerID, ExecutionSlotsUsed: 1, ExecutionSlotsTotal: 1,
|
||||
RunningWork: []*plugin_pb.RunningWork{{
|
||||
WorkId: "lc-1", Kind: plugin_pb.WorkKind_WORK_KIND_EXECUTION, JobType: "s3_lifecycle",
|
||||
}},
|
||||
})
|
||||
// the detection that produced the proposals set the next one a day out
|
||||
daily := time.Now().UTC().Add(24 * time.Hour)
|
||||
pluginSvc.schedulerMu.Lock()
|
||||
pluginSvc.nextDetectionAt["admin_script"] = daily
|
||||
pluginSvc.schedulerMu.Unlock()
|
||||
|
||||
window, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
policy := schedulerPolicy{
|
||||
ExecutionConcurrency: 1,
|
||||
PerWorkerConcurrency: 1,
|
||||
ExecutionTimeout: time.Second,
|
||||
ExecutorReserveBackoff: time.Millisecond,
|
||||
OrphanRehomeGrace: time.Second,
|
||||
}
|
||||
proposals := []*plugin_pb.JobProposal{{ProposalId: "p1", JobType: "admin_script", DedupeKey: "k1"}}
|
||||
if _, _, canceled := pluginSvc.dispatchScheduledProposals(window, "admin_script", proposals, &plugin_pb.ClusterContext{}, policy); canceled != 1 {
|
||||
t.Fatalf("canceled = %d, want 1", canceled)
|
||||
}
|
||||
|
||||
pluginSvc.schedulerMu.Lock()
|
||||
next := pluginSvc.nextDetectionAt["admin_script"]
|
||||
pluginSvc.schedulerMu.Unlock()
|
||||
if latest := time.Now().UTC().Add(capacityGiveUpRedetectDelay); next.After(latest) {
|
||||
t.Fatalf("next detection %v is after %v: a given-up job type waits its whole interval", next, latest)
|
||||
}
|
||||
}
|
||||
|
||||
// Bringing a detection forward never pushes back one already due sooner.
|
||||
func TestBringDetectionForwardKeepsAnEarlierDetection(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
pluginSvc, err := New(Options{})
|
||||
if err != nil {
|
||||
t.Fatalf("New: %v", err)
|
||||
}
|
||||
defer pluginSvc.Shutdown()
|
||||
|
||||
soon := time.Now().UTC().Add(time.Minute)
|
||||
pluginSvc.schedulerMu.Lock()
|
||||
pluginSvc.nextDetectionAt["vacuum"] = soon
|
||||
pluginSvc.schedulerMu.Unlock()
|
||||
|
||||
pluginSvc.bringDetectionForward("vacuum", capacityGiveUpRedetectDelay)
|
||||
|
||||
pluginSvc.schedulerMu.Lock()
|
||||
next := pluginSvc.nextDetectionAt["vacuum"]
|
||||
pluginSvc.schedulerMu.Unlock()
|
||||
if !next.Equal(soon) {
|
||||
t.Fatalf("next detection moved from %v to %v", soon, next)
|
||||
}
|
||||
}
|
||||
@@ -391,6 +391,7 @@ func (lock *LiveLock) AttemptToLock(lockDuration time.Duration) error {
|
||||
|
||||
func (lock *LiveLock) attemptToLockWithin(lockDuration, attemptTimeout time.Duration) (granted bool, err error) {
|
||||
glog.V(4).Infof("LOCK: AttemptToLock key=%s owner=%s", lock.key, lock.self)
|
||||
sentAt := time.Now()
|
||||
errorMessage, granted, err := lock.doLock(lockDuration, attemptTimeout)
|
||||
if err != nil {
|
||||
glog.V(1).Infof("LOCK: doLock failed for key=%s: %v", lock.key, err)
|
||||
@@ -417,6 +418,9 @@ func (lock *LiveLock) attemptToLockWithin(lockDuration, attemptTimeout time.Dura
|
||||
// Only log when transitioning from unlocked to locked
|
||||
glog.V(1).Infof("LOCK: Successfully acquired key=%s owner=%s", lock.key, lock.self)
|
||||
}
|
||||
// The server counts the lease from when it got the request, so a lease
|
||||
// counted from the send never outlives the one the server holds.
|
||||
atomic.StoreInt64(&lock.expireAtNs, sentAt.Add(lockDuration.Truncate(time.Second)).UnixNano())
|
||||
atomic.StoreInt32(&lock.isLocked, 1)
|
||||
return true, nil
|
||||
}
|
||||
@@ -566,7 +570,9 @@ func (lock *LiveLock) Generation() int64 {
|
||||
return atomic.LoadInt64(&lock.generation)
|
||||
}
|
||||
|
||||
// IsLocked returns true if this instance currently holds the lock
|
||||
// IsLocked returns true if this instance currently holds the lock and the
|
||||
// lease of its last grant or renewal has not run out, so a renewal that
|
||||
// stalled past the lease does not read as still holding it.
|
||||
func (lock *LiveLock) IsLocked() bool {
|
||||
return atomic.LoadInt32(&lock.isLocked) == 1
|
||||
return atomic.LoadInt32(&lock.isLocked) == 1 && time.Now().UnixNano() < atomic.LoadInt64(&lock.expireAtNs)
|
||||
}
|
||||
@@ -604,4 +604,51 @@ func TestLiveLockGenerationIsAligned(t *testing.T) {
|
||||
if got := lock.Generation(); got != 42 {
|
||||
t.Fatalf("generation = %d, want 42", got)
|
||||
}
|
||||
atomic.StoreInt64(&lock.expireAtNs, 42)
|
||||
}
|
||||
|
||||
func newGrantingLockClient(t *testing.T) *LockClient {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
grpcServer := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(grpcServer, &grantingLockFiler{})
|
||||
go grpcServer.Serve(listener)
|
||||
t.Cleanup(grpcServer.Stop)
|
||||
|
||||
host, port, err := net.SplitHostPort(listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("split host port: %v", err)
|
||||
}
|
||||
return NewLockClient(grpc.WithTransportCredentials(insecure.NewCredentials()), pb.ServerAddress(fmt.Sprintf("%s:0.%s", host, port)))
|
||||
}
|
||||
|
||||
// A holder whose renewal stalled past the lease is not locked anymore, even
|
||||
// before a failed renewal clears it: the server may have granted the key to
|
||||
// someone else by then.
|
||||
func TestLiveLockIsLockedOnlyWithinLease(t *testing.T) {
|
||||
lock := newGrantingLockClient(t).NewShortLivedLock("test-key", "test-owner")
|
||||
if lock == nil {
|
||||
t.Fatal("lock not granted")
|
||||
}
|
||||
if !lock.IsLocked() {
|
||||
t.Fatal("a fresh grant must read as locked")
|
||||
}
|
||||
if left := time.Until(time.Unix(0, atomic.LoadInt64(&lock.expireAtNs))); left <= 0 || left > 5*time.Second {
|
||||
t.Fatalf("lease left after a 5s grant = %v", left)
|
||||
}
|
||||
|
||||
if err := lock.AttemptToLock(15 * time.Second); err != nil {
|
||||
t.Fatalf("renew: %v", err)
|
||||
}
|
||||
if left := time.Until(time.Unix(0, atomic.LoadInt64(&lock.expireAtNs))); left <= 5*time.Second || left > 15*time.Second {
|
||||
t.Fatalf("lease left after a 15s renewal = %v", left)
|
||||
}
|
||||
|
||||
atomic.StoreInt64(&lock.expireAtNs, time.Now().Add(-time.Millisecond).UnixNano())
|
||||
if lock.IsLocked() {
|
||||
t.Fatal("a lock whose lease ran out must not read as locked")
|
||||
}
|
||||
}
|
||||
@@ -1088,11 +1088,12 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
|
||||
fullpath := util.NewFullPath(req.Directory, req.EntryName)
|
||||
|
||||
lockClient := cluster.NewLockClient(fs.grpcDialOption, fs.option.Host)
|
||||
lock := lockClient.NewShortLivedLock(string(fullpath), string(fs.option.Host))
|
||||
lock := lockClient.NewBlockingLongLivedLock(string(fullpath), string(fs.option.Host), 0)
|
||||
if lock == nil {
|
||||
return nil, fmt.Errorf("failed to acquire lock for %s", fullpath)
|
||||
}
|
||||
defer lock.StopShortLivedLock()
|
||||
defer lock.Stop()
|
||||
generation := lock.Generation()
|
||||
|
||||
// The cluster lock serializes appenders across filers; the path lock makes
|
||||
// this read-modify-write atomic against conditional updates and deletes on
|
||||
@@ -1128,12 +1129,21 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
|
||||
glog.WarningfCtx(ctx, "applyStorageDefaultsToEntry: %v", err)
|
||||
return &filer_pb.AppendToEntryResponse{}, err
|
||||
}
|
||||
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, entry.GetChunks())
|
||||
appended := entry.GetChunks()
|
||||
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, appended)
|
||||
if err != nil {
|
||||
// the append is still correct with the flat list
|
||||
glog.V(0).InfofCtx(ctx, "MaybeManifestize %s: %v", fullpath, err)
|
||||
}
|
||||
|
||||
// Best effort: a lock lost, or lost and taken again, since the entry was
|
||||
// read means another filer may have appended in between. The store write
|
||||
// is not conditional, so a lapse after this check still goes unnoticed.
|
||||
if !lock.IsLocked() || lock.Generation() != generation {
|
||||
fs.filer.DeleteChunksNotRecursive(filer.DoMinusChunks(entry.GetChunks(), appended))
|
||||
return &filer_pb.AppendToEntryResponse{}, fmt.Errorf("lost distributed lock for %s; retry the request", fullpath)
|
||||
}
|
||||
|
||||
err = fs.filer.CreateEntry(context.Background(), entry, nil, false, false, nil, false, fs.filer.MaxFilenameLength)
|
||||
|
||||
return &filer_pb.AppendToEntryResponse{}, err
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
const appendLockTestPath = "/data/app.log"
|
||||
|
||||
// appendHoldStore parks the first lookup of appendLockTestPath, which
|
||||
// AppendToEntry makes while it holds the cluster lock, until released.
|
||||
type appendHoldStore struct {
|
||||
*renameTestStore
|
||||
once sync.Once
|
||||
held chan struct{}
|
||||
release chan struct{}
|
||||
}
|
||||
|
||||
func (s *appendHoldStore) FindEntry(ctx context.Context, p util.FullPath) (*filer.Entry, error) {
|
||||
if string(p) == appendLockTestPath {
|
||||
s.once.Do(func() {
|
||||
close(s.held)
|
||||
<-s.release
|
||||
})
|
||||
}
|
||||
return s.renameTestStore.FindEntry(ctx, p)
|
||||
}
|
||||
|
||||
// newAppendLockServer serves the filer's lock RPCs on a real listener so
|
||||
// AppendToEntry takes its cluster lock from itself, as a one-filer ring does.
|
||||
func newAppendLockServer(t *testing.T) (*FilerServer, *appendHoldStore) {
|
||||
store := &appendHoldStore{
|
||||
renameTestStore: newRenameTestStore(),
|
||||
held: make(chan struct{}),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
f := newRenameTestFiler(t, store.renameTestStore)
|
||||
f.Store = filer.NewFilerStoreWrapper(store)
|
||||
fs := &FilerServer{
|
||||
filer: f,
|
||||
option: &FilerOption{},
|
||||
entryLockTable: util.NewLockTable[util.FullPath](),
|
||||
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
}
|
||||
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
addr := pb.NewServerAddressWithGrpcPort(lis.Addr().String(), lis.Addr().(*net.TCPAddr).Port)
|
||||
withRing(fs, addr, addr)
|
||||
srv := grpc.NewServer()
|
||||
filer_pb.RegisterSeaweedFilerServer(srv, fs)
|
||||
go srv.Serve(lis)
|
||||
t.Cleanup(srv.Stop)
|
||||
return fs, store
|
||||
}
|
||||
|
||||
func startAppend(fs *FilerServer) chan error {
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := fs.AppendToEntry(context.Background(), &filer_pb.AppendToEntryRequest{
|
||||
Directory: "/data",
|
||||
EntryName: "app.log",
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 10}},
|
||||
})
|
||||
done <- err
|
||||
}()
|
||||
return done
|
||||
}
|
||||
|
||||
// An append that outlasts the lock's first lease keeps the lock: another
|
||||
// filer must not be able to take it and append concurrently.
|
||||
func TestAppendToEntryKeepsLockThroughSlowAppend(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs, store := newAppendLockServer(t)
|
||||
done := startAppend(fs)
|
||||
<-store.held
|
||||
|
||||
time.Sleep(lock_manager.LiveLockTTL + time.Second) // past the first lease
|
||||
resp, err := fs.DistributedLock(context.Background(), &filer_pb.LockRequest{
|
||||
Name: appendLockTestPath,
|
||||
SecondsToLock: 5,
|
||||
Owner: "other-filer",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Contains(t, resp.Error, "lock already owned", "another filer took the lock of a running append")
|
||||
|
||||
close(store.release)
|
||||
require.NoError(t, <-done)
|
||||
require.Len(t, store.entries[appendLockTestPath].GetChunks(), 1)
|
||||
}
|
||||
|
||||
// takeAppendLock hands the append's lock to another filer, as a lease that
|
||||
// expired during a stall and was granted elsewhere would.
|
||||
func takeAppendLock(fs *FilerServer, expiresIn time.Duration) {
|
||||
fs.filer.Dlm.InsertLock(appendLockTestPath, time.Now().Add(expiresIn).UnixNano(), "other-token", "", "other-filer", 1<<30, 1)
|
||||
}
|
||||
|
||||
// An append whose lock renewal finds another owner must not commit.
|
||||
func TestAppendToEntryAbortsAfterLosingLock(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs, store := newAppendLockServer(t)
|
||||
done := startAppend(fs)
|
||||
<-store.held
|
||||
|
||||
takeAppendLock(fs, time.Minute)
|
||||
time.Sleep(5 * time.Second) // past the first renewal and its retry pause
|
||||
close(store.release)
|
||||
|
||||
require.ErrorContains(t, <-done, "lost distributed lock")
|
||||
require.NotContains(t, store.entries, appendLockTestPath)
|
||||
}
|
||||
|
||||
// A lock lost and then granted again under a new generation still lost its
|
||||
// exclusion in between, so the append must not commit either.
|
||||
func TestAppendToEntryAbortsAfterRetakingLock(t *testing.T) {
|
||||
t.Parallel()
|
||||
fs, store := newAppendLockServer(t)
|
||||
done := startAppend(fs)
|
||||
<-store.held
|
||||
|
||||
takeAppendLock(fs, 2*time.Second)
|
||||
time.Sleep(8 * time.Second) // the first renewal fails, the second takes the lock again
|
||||
close(store.release)
|
||||
|
||||
require.ErrorContains(t, <-done, "lost distributed lock")
|
||||
require.NotContains(t, store.entries, appendLockTestPath)
|
||||
}
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
)
|
||||
|
||||
const recentCopyRequestTTL = 2 * time.Minute
|
||||
@@ -262,7 +263,12 @@ func (fs *FilerServer) copyEntry(ctx context.Context, srcEntry *filer.Entry, dst
|
||||
srcChunks := srcEntry.GetChunks()
|
||||
|
||||
// Create HTTP client once for reuse across all chunk operations
|
||||
client := &http.Client{Timeout: 60 * time.Second}
|
||||
globalClient := util_http.GetGlobalHttpClient()
|
||||
client := &http.Client{
|
||||
Timeout: 60 * time.Second,
|
||||
Transport: globalClient.GetClientTransport(),
|
||||
CheckRedirect: globalClient.Client.CheckRedirect,
|
||||
}
|
||||
|
||||
// Check if any chunks are manifest chunks - these require special handling
|
||||
if filer.HasChunkManifest(srcChunks) {
|
||||
@@ -684,7 +690,11 @@ func (fs *FilerServer) createManifestChunk(ctx context.Context, dataChunks []*fi
|
||||
|
||||
// uploadData uploads data to a volume server
|
||||
func (fs *FilerServer) uploadData(ctx context.Context, reader io.Reader, urlLocation, auth string, client *http.Client) error {
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", urlLocation, reader)
|
||||
target, err := util_http.NormalizeUrl(urlLocation)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid upload url %s: %w", urlLocation, err)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "PUT", target, reader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create upload request: %w", err)
|
||||
}
|
||||
@@ -765,7 +775,7 @@ func (fs *FilerServer) streamCopyChunk(ctx context.Context, srcChunk *filer_pb.F
|
||||
glog.V(4).InfofCtx(ctx, "FilerServer.streamCopyChunk: attempting streaming copy from %s to %s (attempt %d/%d)", srcUrl, urlLocation, i+1, len(locations))
|
||||
|
||||
// Perform streaming copy using HTTP client
|
||||
err := fs.performStreamCopy(ctx, srcUrl, urlLocation, string(auth), srcChunk.Size, client)
|
||||
err := fs.performStreamCopy(ctx, srcUrl, filer.JwtForVolumeServer(fileIdString), urlLocation, string(auth), srcChunk.Size, client)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
glog.V(2).InfofCtx(ctx, "FilerServer.streamCopyChunk: failed streaming copy from %s: %v", srcUrl, err)
|
||||
@@ -789,12 +799,23 @@ func (fs *FilerServer) streamCopyChunk(ctx context.Context, srcChunk *filer_pb.F
|
||||
}
|
||||
|
||||
// performStreamCopy performs the actual streaming copy from source URL to destination URL
|
||||
func (fs *FilerServer) performStreamCopy(ctx context.Context, srcUrl, dstUrl, auth string, expectedSize uint64, client *http.Client) error {
|
||||
func (fs *FilerServer) performStreamCopy(ctx context.Context, srcUrl, srcAuth, dstUrl, auth string, expectedSize uint64, client *http.Client) error {
|
||||
srcUrl, err := util_http.NormalizeUrl(srcUrl)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid source url: %v", err)
|
||||
}
|
||||
if dstUrl, err = util_http.NormalizeUrl(dstUrl); err != nil {
|
||||
return fmt.Errorf("invalid destination url: %v", err)
|
||||
}
|
||||
|
||||
// Create HTTP request to read from source
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", srcUrl, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create source request: %v", err)
|
||||
}
|
||||
if srcAuth != "" {
|
||||
req.Header.Set("Authorization", security.BearerPrefix+srcAuth)
|
||||
}
|
||||
|
||||
// Perform source request
|
||||
resp, err := client.Do(req)
|
||||
|
||||
@@ -2,6 +2,10 @@ package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -221,3 +225,38 @@ func TestRecentCopyRequestDeduplicatesByRequestID(t *testing.T) {
|
||||
t.Fatal("reused request id with different fingerprint should not be treated as handled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerformStreamCopySendsSourceReadToken(t *testing.T) {
|
||||
src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer read-tok" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
io.WriteString(w, "data")
|
||||
}))
|
||||
defer src.Close()
|
||||
var written string
|
||||
dst := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Authorization") != "Bearer write-tok" {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
written = string(b)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
}))
|
||||
defer dst.Close()
|
||||
|
||||
fs := &FilerServer{}
|
||||
ctx := context.Background()
|
||||
if err := fs.performStreamCopy(ctx, src.URL+"/3,01637037d6", "read-tok", dst.URL+"/4,01637037d7", "write-tok", 4, http.DefaultClient); err != nil {
|
||||
t.Fatalf("copy with read token: %v", err)
|
||||
}
|
||||
if written != "data" {
|
||||
t.Fatalf("destination got %q, want %q", written, "data")
|
||||
}
|
||||
err := fs.performStreamCopy(ctx, src.URL+"/3,01637037d6", "", dst.URL+"/4,01637037d7", "write-tok", 4, http.DefaultClient)
|
||||
if err == nil || !strings.Contains(err.Error(), "401") {
|
||||
t.Fatalf("copy without read token: got %v, want a 401 from the source", err)
|
||||
}
|
||||
}
|
||||
@@ -440,8 +440,11 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
// determine the scheme based on HTTPS client configuration
|
||||
scheme := util_http.GetGlobalHttpClient().GetHttpScheme()
|
||||
// the leader serves HTTPS exactly when this master does
|
||||
scheme := "http"
|
||||
if util.GetViper().GetString("https.master.key") != "" {
|
||||
scheme = "https"
|
||||
}
|
||||
|
||||
targetUrl, err := url.Parse(scheme + "://" + raftServerLeader)
|
||||
if err != nil {
|
||||
|
||||
Reference in new issue
Block a user