Compare commits

...
Author SHA1 Message Date
Chris LuandClaude Opus 5.5 c7287f8f54 filer: send a read token on the cp.from source chunk GET
With jwt.signing.read set, volume servers refuse unsigned reads, so
cp.from of a chunked file failed with "source returned status 401".
Mint a read token for the chunk on each source attempt, the same way the
filer's own reads do.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 04:03:29 -08:00
Chris LuandClaude Opus 5.5 e00698a327 helm: update the volume HTTPS caveats now that masters forward correctly
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:56:58 -08:00
Chris LuandClaude Opus 5.5 63a575f28f master: forward to the leader with the masters' own HTTP scheme
A follower took the scheme from [https.client], so with HTTPS clients
toward volume servers and plain-HTTP masters every forwarded request
(/dir/assign, /dir/status, the UI) failed with 502 "first record does not
look like a TLS handshake". Masters share one [https.master] setting, so
use HTTPS exactly when this master serves it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:56:47 -08:00
Chris LuandClaude Opus 5.5 ed67f9930c helm CI: check the volume HTTPS rollout annotation
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:44:01 -08:00
Chris LuandClaude Opus 5.5 dd4d06e40f helm: restart every security.toml reader when volume HTTPS is toggled
security.toml is mounted with subPath and read once at startup, so on an
upgrade that turns volume HTTPS on, the volume servers restarted with TLS
while filer, S3 and the other clients kept speaking plain HTTP. A
checksum/volume-https pod annotation, rendered only while the option is
on, rolls them all.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:44:00 -08:00
Chris LuandClaude Opus 5.5 64d5fb5613 filer: copy chunks for cp.from through the shared HTTP client
The copy built a bare client and plain http:// chunk URLs, so with
[https.client] on it could not read from or write to HTTPS volume servers
("source returned status 400"). Use the global client's transport and
redirect policy, and normalize the source and destination URLs to its
scheme.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:42:22 -08:00
Chris LuandClaude Opus 5.5 e15aa250e9 helm: document volume server HTTPS
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:25:11 -08:00
Chris LuandClaude Opus 5.5 c908497d56 helm CI: check the volume server HTTPS rendering and its guards
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:24:09 -08:00
Chris LuandClaude Opus 5.5 9bc1a50b37 helm: reject volume HTTPS for the all-in-one deployment
Its volume server registers by pod IP, so HTTPS clients could not verify
the volume certificate.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:23:40 -08:00
Chris LuandClaude Opus 5.5 dc2d13a1ee helm: probe volume servers over HTTPS when they serve it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:23:29 -08:00
Chris LuandClaude Opus 5.5 af35fb158c helm: add the volume service names to the volume certificate for HTTPS
Volume servers register as <pod>.<volume service>.<namespace>, and HTTPS
clients verify that name, which none of the certificate's wildcards cover.
With volume HTTPS on, the certificate also lists every volume group's
service.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:23:12 -08:00
Chris LuandClaude Opus 5.5 080af98b86 helm: render volume server HTTPS in security.toml
global.seaweedfs.volumeHttps.enabled sets [https.volume] to the volume
certificate and turns on [https.client] with the client certificate and the
chart CA, so the volume data port is no longer plaintext under
enableSecurity. The option fails without enableSecurity.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-20 03:22:53 -08:00
19 changed files with 182 additions and 11 deletions

No files matched your search

+34
View File
@@ -259,6 +259,40 @@ jobs:
grep -q "security-config" /tmp/security.yaml
echo "Security configuration renders correctly"
echo "=== Testing volume server HTTPS ==="
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/security.yaml | grep -q 'enabled = false'
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set global.seaweedfs.volumeHttps.enabled=true \
--set volumes.hot.replicas=1 \
> /tmp/volume-https.yaml
awk '/^ *\[https.client\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml > /tmp/volume-https-client.txt
grep -q 'enabled = true' /tmp/volume-https-client.txt
grep -q 'cert = "/usr/local/share/ca-certificates/client/tls.crt"' /tmp/volume-https-client.txt
grep -q 'ca = "/usr/local/share/ca-certificates/ca/tls.crt"' /tmp/volume-https-client.txt
awk '/^ *\[https.volume\]$/{f=1;next} /^ *\[/{f=0} f' /tmp/volume-https.yaml \
| grep -q 'cert = "/usr/local/share/ca-certificates/volume/tls.crt"'
grep -q "'\*.test-seaweedfs-volume.default'" /tmp/volume-https.yaml
grep -q "'\*.test-seaweedfs-volume-hot.default'" /tmp/volume-https.yaml
test "$(grep -c 'scheme: HTTPS' /tmp/volume-https.yaml)" -eq 4
test "$(grep -c 'checksum/volume-https:' /tmp/volume-https.yaml)" -eq 4
if grep -q 'checksum/volume-https' /tmp/security.yaml; then
echo "FAIL: the volume HTTPS rollout annotation should only render while the option is on"
exit 1
fi
if helm template test $CHART_DIR --set global.seaweedfs.volumeHttps.enabled=true > /dev/null 2>/tmp/volume-https-nosec.err; then
echo "FAIL: volumeHttps without enableSecurity should fail to render"
exit 1
fi
grep -q "requires global.seaweedfs.enableSecurity" /tmp/volume-https-nosec.err
if helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true \
--set global.seaweedfs.volumeHttps.enabled=true --set allInOne.enabled=true > /dev/null 2>/tmp/volume-https-aio.err; then
echo "FAIL: volumeHttps with allInOne should fail to render"
exit 1
fi
grep -q "not supported with allInOne" /tmp/volume-https-aio.err
echo "Volume server HTTPS renders [https.volume], [https.client], certificate names, HTTPS probes and rollout annotations"
echo "=== Testing secure bucket-creation hook certificate mounts ==="
helm template test $CHART_DIR \
--show-only templates/shared/post-install-bucket-hook.yaml \
+11
View File
@@ -612,6 +612,17 @@ Two things worth knowing before you turn this on:
The DNS selectors default to CoreDNS as kubeadm, kind and the managed offerings from AWS, Google and Azure install it. On OpenShift, override `egress.dnsNamespaceSelector` and `egress.dnsPodSelector` to match `openshift-dns`; see the comment in `values.yaml`.
## Volume server HTTPS
With `global.seaweedfs.enableSecurity`, gRPC between components uses mutual TLS, but the volume servers' HTTP port, which carries the file data, stays plaintext. Set `global.seaweedfs.volumeHttps.enabled: true` to serve it over TLS:
- `[https.volume]` points at the volume certificate, which then also names every volume group's service (`*.<fullname>-volume.<namespace>`), since volume servers register as `<pod>.<fullname>-volume.<namespace>`. With `certificates.externalCertificates`, the provided `<fullname>-volume-cert` needs those names.
- `[https.client]` is enabled with the client certificate and the chart CA, so every chart pod that talks to the volume servers does it over HTTPS.
- The volume probes use HTTPS.
- Every pod that reads `security.toml` gets a `checksum/volume-https` annotation, so switching the option on or off restarts them together.
The volume servers do not require client certificates, because kubelet probes cannot present one; enable JWT signing (`securityConfig.jwtSigning`) to authorize volume reads and writes. `[https.client]` also sends the admin UI's HTTP lookups on masters and S3 over HTTPS, which their plain listeners refuse, so the UI can miss volume public URLs and refuse SSE-S3 vector buckets. A volume Ingress or HTTPRoute needs its controller's HTTPS-backend setting. `weed mount`, the CSI driver and other clients outside the chart need their own `[https.client]` with the chart CA. The option is not supported with `allInOne`, whose volume server registers by pod IP.
## Common labels
`commonLabels` adds labels to the metadata of every object the chart creates. It may be a map or a template string rendering to one. The chart's own labels cannot be overridden, and pods (see `podLabels`) and `volumeClaimTemplates` are left unchanged.
@@ -56,6 +56,7 @@ spec:
{{- with .Values.admin.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.admin.restartPolicy }}
{{- if .Values.admin.affinity }}
@@ -1,5 +1,8 @@
{{- include "seaweedfs.compat" . -}}
{{- if .Values.allInOne.enabled }}
{{- if include "seaweedfs.volumeHttps" . }}
{{- fail "global.seaweedfs.volumeHttps is not supported with allInOne: its volume server registers by pod IP, which the volume certificate cannot name" }}
{{- end }}
{{- $s3PreStopSeconds := 0 }}
{{- if .Values.allInOne.s3.enabled }}
{{- $s3PreStopSeconds = .Values.allInOne.s3.preStopSeconds }}
@@ -38,6 +38,14 @@ spec:
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}'
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc'
- '*.{{ include "seaweedfs.fullname" . }}-master.{{ include "seaweedfs.namespace" . }}.svc.cluster.local'
{{- if include "seaweedfs.volumeHttps" . }}
{{- range $vname, $_ := deepCopy .Values.volumes | mergeOverwrite (dict "" .Values.volume) }}
{{- $volumeService := printf "%s.%s" (include "seaweedfs.componentName" (list $ (trimSuffix "-" (printf "volume-%s" $vname)))) (include "seaweedfs.namespace" $) }}
- '*.{{ $volumeService }}'
- '*.{{ $volumeService }}.svc'
- '*.{{ $volumeService }}.svc.cluster.local'
{{- end }}
{{- end }}
{{- if .Values.certificates.ipAddresses }}
ipAddresses:
{{- range .Values.certificates.ipAddresses }}
@@ -39,6 +39,7 @@ spec:
{{- with .Values.cosi.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.cosi.restartPolicy }}
{{- if .Values.cosi.affinity }}
@@ -57,6 +57,7 @@ spec:
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.filer.restartPolicy }}
{{- if .Values.filer.affinity }}
@@ -48,6 +48,7 @@ spec:
{{- $_ := set $podAnnotations "checksum/master-config" (include "seaweedfs.master.configChecksum" .) }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.master.restartPolicy }}
{{- if .Values.master.affinity }}
@@ -54,6 +54,7 @@ spec:
{{- end }}
annotations:
{{- toYaml $podAnnotations | nindent 8 }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.s3.restartPolicy }}
{{- if .Values.s3.affinity }}
@@ -43,6 +43,7 @@ spec:
{{- with .Values.sftp.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.sftp.restartPolicy }}
{{- if .Values.sftp.affinity }}
@@ -491,6 +491,25 @@ true
{{- end -}}
{{- end -}}
{{/* True when the volume servers serve HTTPS. Their certificates come from
enableSecurity, so the option fails without it. */}}
{{- define "seaweedfs.volumeHttps" -}}
{{- if ((.Values.global.seaweedfs).volumeHttps).enabled -}}
{{- if not .Values.global.seaweedfs.enableSecurity -}}
{{- fail "global.seaweedfs.volumeHttps.enabled requires global.seaweedfs.enableSecurity, which provides the certificates" -}}
{{- end -}}
true
{{- end -}}
{{- end -}}
{{/* Pod annotation that restarts the readers of security.toml when volume
HTTPS is switched on or off. */}}
{{- define "seaweedfs.volumeHttpsAnnotation" -}}
{{- with include "seaweedfs.volumeHttps" . -}}
checksum/volume-https: {{ sha256sum . | quote }}
{{- end -}}
{{- end -}}
{{/* Read a JWT key from the chart's existing security.toml without fromToml
(which requires Helm >=3.17). Args: (list "<section>" $raw).
Return the single-line TOML string token, including its quotes, so escapes
@@ -121,13 +121,25 @@ data:
cert = "/usr/local/share/ca-certificates/client/tls.crt"
key = "/usr/local/share/ca-certificates/client/tls.key"
# volume server https options
# Note: work in progress!
# this does not work with other clients, e.g., "weed filer|mount" etc, yet.
{{- if include "seaweedfs.volumeHttps" . }}
[https.client]
enabled = true
cert = "/usr/local/share/ca-certificates/client/tls.crt"
key = "/usr/local/share/ca-certificates/client/tls.key"
ca = "/usr/local/share/ca-certificates/ca/tls.crt"
[https.volume]
cert = "/usr/local/share/ca-certificates/volume/tls.crt"
key = "/usr/local/share/ca-certificates/volume/tls.key"
{{- else }}
# volume server https: set global.seaweedfs.volumeHttps.enabled
[https.client]
enabled = false
[https.volume]
cert = ""
key = ""
{{- end }}
{{- end }}
{{- end }}
@@ -25,6 +25,7 @@ spec:
annotations:
{{- with .Values.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
{{- with .Values.vector.podAnnotations }}{{ toYaml . | nindent 8 }}{{ end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
{{- include "seaweedfs.imagePullSecrets" . | nindent 6 }}
automountServiceAccountToken: {{ .Values.global.seaweedfs.automountServiceAccountToken }}
@@ -55,6 +55,7 @@ spec:
{{- with $volume.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
{{- if $volume.affinity }}
affinity:
@@ -281,7 +282,7 @@ spec:
httpGet:
path: {{ $volume.readinessProbe.httpGet.path }}
port: {{ $volume.port }}
scheme: {{ $volume.readinessProbe.httpGet.scheme }}
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.readinessProbe.httpGet.scheme }}{{ end }}
initialDelaySeconds: {{ $volume.readinessProbe.initialDelaySeconds }}
periodSeconds: {{ $volume.readinessProbe.periodSeconds }}
successThreshold: {{ $volume.readinessProbe.successThreshold }}
@@ -293,7 +294,7 @@ spec:
httpGet:
path: {{ $volume.livenessProbe.httpGet.path }}
port: {{ $volume.port }}
scheme: {{ $volume.livenessProbe.httpGet.scheme }}
scheme: {{ if include "seaweedfs.volumeHttps" $ }}HTTPS{{ else }}{{ $volume.livenessProbe.httpGet.scheme }}{{ end }}
initialDelaySeconds: {{ $volume.livenessProbe.initialDelaySeconds }}
periodSeconds: {{ $volume.livenessProbe.periodSeconds }}
successThreshold: {{ $volume.livenessProbe.successThreshold }}
@@ -46,6 +46,7 @@ spec:
{{- with .Values.worker.podAnnotations }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with include "seaweedfs.volumeHttpsAnnotation" $ }}{{ . | nindent 8 }}{{ end }}
spec:
restartPolicy: {{ default .Values.global.seaweedfs.restartPolicy .Values.worker.restartPolicy }}
{{- if .Values.worker.affinity }}
+12
View File
@@ -45,6 +45,18 @@ global:
volumeRead: 0
filerWrite: 0
filerRead: 0
# Serve the volume servers' HTTP port over TLS with the chart's volume
# certificate, and make every chart component reach them over HTTPS
# ([https.volume] and [https.client] in security.toml). Requires
# enableSecurity and is not supported with allInOne. Clients outside the
# chart (weed mount, the CSI driver) need their own [https.client].
# The volume servers do not ask for client certificates, because kubelet
# probes cannot present one; JWT signing keeps authorizing volume access.
# [https.client] also sends the admin UI's HTTP lookups on masters and
# S3 over HTTPS, which their plain listeners refuse, so the UI can miss
# volume public URLs and refuse SSE-S3 vector buckets.
volumeHttps:
enabled: false
# we will use this serviceAccountName for all ClusterRoles/ClusterRoleBindings
serviceAccountName: "seaweedfs"
serviceAccountAnnotations: {}
+25 -4
View File
@@ -20,6 +20,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
)
const recentCopyRequestTTL = 2 * time.Minute
@@ -262,7 +263,12 @@ func (fs *FilerServer) copyEntry(ctx context.Context, srcEntry *filer.Entry, dst
srcChunks := srcEntry.GetChunks()
// Create HTTP client once for reuse across all chunk operations
client := &http.Client{Timeout: 60 * time.Second}
globalClient := util_http.GetGlobalHttpClient()
client := &http.Client{
Timeout: 60 * time.Second,
Transport: globalClient.GetClientTransport(),
CheckRedirect: globalClient.Client.CheckRedirect,
}
// Check if any chunks are manifest chunks - these require special handling
if filer.HasChunkManifest(srcChunks) {
@@ -684,7 +690,11 @@ func (fs *FilerServer) createManifestChunk(ctx context.Context, dataChunks []*fi
// uploadData uploads data to a volume server
func (fs *FilerServer) uploadData(ctx context.Context, reader io.Reader, urlLocation, auth string, client *http.Client) error {
req, err := http.NewRequestWithContext(ctx, "PUT", urlLocation, reader)
target, err := util_http.NormalizeUrl(urlLocation)
if err != nil {
return fmt.Errorf("invalid upload url %s: %w", urlLocation, err)
}
req, err := http.NewRequestWithContext(ctx, "PUT", target, reader)
if err != nil {
return fmt.Errorf("failed to create upload request: %w", err)
}
@@ -765,7 +775,7 @@ func (fs *FilerServer) streamCopyChunk(ctx context.Context, srcChunk *filer_pb.F
glog.V(4).InfofCtx(ctx, "FilerServer.streamCopyChunk: attempting streaming copy from %s to %s (attempt %d/%d)", srcUrl, urlLocation, i+1, len(locations))
// Perform streaming copy using HTTP client
err := fs.performStreamCopy(ctx, srcUrl, urlLocation, string(auth), srcChunk.Size, client)
err := fs.performStreamCopy(ctx, srcUrl, filer.JwtForVolumeServer(fileIdString), urlLocation, string(auth), srcChunk.Size, client)
if err != nil {
lastErr = err
glog.V(2).InfofCtx(ctx, "FilerServer.streamCopyChunk: failed streaming copy from %s: %v", srcUrl, err)
@@ -789,12 +799,23 @@ func (fs *FilerServer) streamCopyChunk(ctx context.Context, srcChunk *filer_pb.F
}
// performStreamCopy performs the actual streaming copy from source URL to destination URL
func (fs *FilerServer) performStreamCopy(ctx context.Context, srcUrl, dstUrl, auth string, expectedSize uint64, client *http.Client) error {
func (fs *FilerServer) performStreamCopy(ctx context.Context, srcUrl, srcAuth, dstUrl, auth string, expectedSize uint64, client *http.Client) error {
srcUrl, err := util_http.NormalizeUrl(srcUrl)
if err != nil {
return fmt.Errorf("invalid source url: %v", err)
}
if dstUrl, err = util_http.NormalizeUrl(dstUrl); err != nil {
return fmt.Errorf("invalid destination url: %v", err)
}
// Create HTTP request to read from source
req, err := http.NewRequestWithContext(ctx, "GET", srcUrl, nil)
if err != nil {
return fmt.Errorf("failed to create source request: %v", err)
}
if srcAuth != "" {
req.Header.Set("Authorization", security.BearerPrefix+srcAuth)
}
// Perform source request
resp, err := client.Do(req)
@@ -2,6 +2,10 @@ package weed_server
import (
"context"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
@@ -221,3 +225,38 @@ func TestRecentCopyRequestDeduplicatesByRequestID(t *testing.T) {
t.Fatal("reused request id with different fingerprint should not be treated as handled")
}
}
func TestPerformStreamCopySendsSourceReadToken(t *testing.T) {
src := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer read-tok" {
w.WriteHeader(http.StatusUnauthorized)
return
}
io.WriteString(w, "data")
}))
defer src.Close()
var written string
dst := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer write-tok" {
w.WriteHeader(http.StatusUnauthorized)
return
}
b, _ := io.ReadAll(r.Body)
written = string(b)
w.WriteHeader(http.StatusCreated)
}))
defer dst.Close()
fs := &FilerServer{}
ctx := context.Background()
if err := fs.performStreamCopy(ctx, src.URL+"/3,01637037d6", "read-tok", dst.URL+"/4,01637037d7", "write-tok", 4, http.DefaultClient); err != nil {
t.Fatalf("copy with read token: %v", err)
}
if written != "data" {
t.Fatalf("destination got %q, want %q", written, "data")
}
err := fs.performStreamCopy(ctx, src.URL+"/3,01637037d6", "", dst.URL+"/4,01637037d7", "write-tok", 4, http.DefaultClient)
if err == nil || !strings.Contains(err.Error(), "401") {
t.Fatalf("copy without read token: got %v, want a 401 from the source", err)
}
}
+5 -2
View File
@@ -440,8 +440,11 @@ func (ms *MasterServer) proxyToLeader(f http.HandlerFunc) http.HandlerFunc {
return
}
// determine the scheme based on HTTPS client configuration
scheme := util_http.GetGlobalHttpClient().GetHttpScheme()
// the leader serves HTTPS exactly when this master does
scheme := "http"
if util.GetViper().GetString("https.master.key") != "" {
scheme = "https"
}
targetUrl, err := url.Parse(scheme + "://" + raftServerLeader)
if err != nil {