Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-20 13:30:46 +02:00
Code Issues Packages Projects Releases Wiki Activity
Files
2d41fb255f6a8e1e32f0df0e3452e600ee7721d9
seaweedfs/.github/workflows/helm_ci.yml
T
Sebastian PreisnerandClaude Opus 5 2d41fb255f helm: refuse empty port and DNS peer lists instead of widening 
In a NetworkPolicy an empty ports list means every port and a missing peer
selector means every pod, so `kubeApiServer.ports: []` silently opened the
API server CIDRs on all ports, and nulling a DNS selector rendered
`podSelector: null`, which is every pod in kube-system. Both now fail the
render, and the DNS rule emits only the selectors that are set.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-29 11:59:38 +02:00

1192 lines
58 KiB
YAML
Raw Blame History

name: "helm: lint and test charts"
on:
push:
branches: [ master ]
paths: ['k8s/**']
pull_request:
branches: [ master ]
paths: ['k8s/**']
permissions:
contents: read
jobs:
lint-test:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
fetch-depth: 0
- name: Set up Helm
uses: azure/setup-helm@v5
with:
version: v3.18.4
- uses: actions/setup-python@v7
with:
python-version: '3.10'
check-latest: true
- name: Set up chart-testing
uses: helm/chart-testing-action@v2.8.0
- name: Run chart-testing (list-changed)
id: list-changed
run: |
changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }} --chart-dirs k8s/charts)
if [[ -n "$changed" ]]; then
echo "::set-output name=changed::true"
fi
- name: Run chart-testing (lint)
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --chart-dirs k8s/charts
- name: Verify template rendering
run: |
set -e
CHART_DIR="k8s/charts/seaweedfs"
echo "=== Testing default configuration ==="
helm template test $CHART_DIR > /tmp/default.yaml
echo "Default configuration renders successfully"
echo "=== Testing with S3 enabled ==="
helm template test $CHART_DIR --set s3.enabled=true > /tmp/s3.yaml
grep -q "kind: Deployment" /tmp/s3.yaml && grep -q "seaweedfs-s3" /tmp/s3.yaml
echo "S3 deployment renders correctly"
echo "=== Testing with all-in-one mode ==="
helm template test $CHART_DIR --set allInOne.enabled=true > /tmp/allinone.yaml
grep -q "seaweedfs-all-in-one" /tmp/allinone.yaml
echo "All-in-one deployment renders correctly"
echo "=== Testing with security enabled ==="
helm template test $CHART_DIR --set global.seaweedfs.enableSecurity=true > /tmp/security.yaml
grep -q "security-config" /tmp/security.yaml
echo "Security configuration renders correctly"
echo ""
echo "=== Testing JWT expiration overrides ==="
helm template test $CHART_DIR \
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
> /tmp/jwt-volume-write-expiration.yaml
grep -q "security-config" /tmp/jwt-volume-write-expiration.yaml
grep -q "expires_after_seconds = 11" /tmp/jwt-volume-write-expiration.yaml
helm template test $CHART_DIR \
--set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
--set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
--set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeWrite=11 \
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.volumeRead=22 \
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerWrite=33 \
--set global.seaweedfs.securityConfig.jwtSigning.expiresAfterSeconds.filerRead=44 \
> /tmp/jwt-expiration.yaml
assert_jwt_expiration() {
local section="$1"
local seconds="$2"
awk -v section="[$section]" -v seconds="$seconds" '
/^[[:space:]]*\[.*\][[:space:]]*$/ {
in_section = index($0, section) > 0
}
in_section && $0 ~ "^[[:space:]]*expires_after_seconds = " seconds "$" {
found = 1
}
END { exit !found }
' /tmp/jwt-expiration.yaml
}
assert_jwt_expiration jwt.signing 11
assert_jwt_expiration jwt.signing.read 22
assert_jwt_expiration jwt.filer_signing 33
assert_jwt_expiration jwt.filer_signing.read 44
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set global.seaweedfs.securityConfig.jwtSigning.volumeRead=true \
--set global.seaweedfs.securityConfig.jwtSigning.filerWrite=true \
--set global.seaweedfs.securityConfig.jwtSigning.filerRead=true \
> /tmp/jwt-default-expiration.yaml
if grep -q "expires_after_seconds =" /tmp/jwt-default-expiration.yaml; then
echo "FAIL: zero JWT expiration values should preserve runtime defaults"
exit 1
fi
echo "JWT expiration overrides render correctly"
echo ""
echo "=== Testing IAM gRPC opt-in path ==="
# Regression test: the filer registers the IAM gRPC service the
# Admin UI Users tab calls only when jwt.filer_signing.key is in
# security.toml. Operators must be able to enable that without
# the cert-manager mTLS bundle.
# Install PyYAML explicitly: this block runs before the later
# security+S3 block that does the same install, and we don't
# want to rely on the runner image shipping it.
pip install pyyaml -q
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def docs(manifest):
return [d for d in yaml.safe_load_all(manifest) if d]
def configmap(manifest, name):
for d in docs(manifest):
if d.get("kind") == "ConfigMap" and d["metadata"]["name"] == name:
return d
return None
def workload_mounts(manifest, name):
for d in docs(manifest):
if d.get("kind") not in ("Deployment", "StatefulSet"):
continue
if d["metadata"]["name"] != name:
continue
pod = d["spec"]["template"]["spec"]
vols = {v["name"] for v in pod.get("volumes", [])}
mounts = set()
for c in pod.get("containers", []):
for vm in c.get("volumeMounts", []):
mounts.add(vm["name"])
return vols, mounts
return None, None
failed = []
# Case 1: defaults. The chart historically rendered nothing
# security-related; preserve that so this PR is non-breaking on
# existing installs.
out = render({})
if configmap(out, "test-seaweedfs-security-config") is not None:
failed.append("defaults: security ConfigMap should not render")
else:
print("defaults: no security-config ConfigMap (unchanged)")
# Case 2: filerWrite=true alone is the documented opt-in for
# the Admin UI Users tab. Configmap must render with
# [jwt.filer_signing] and NO [grpc.*] sections (cert paths
# only exist with mTLS).
out = render({
"global.seaweedfs.securityConfig.jwtSigning.filerWrite": "true",
"admin.enabled": "true",
})
cm = configmap(out, "test-seaweedfs-security-config")
if cm is None:
failed.append("filerWrite=true: security ConfigMap missing")
else:
toml = cm["data"]["security.toml"]
if "[jwt.filer_signing]" not in toml:
failed.append("filerWrite=true: security.toml missing [jwt.filer_signing]")
if "[grpc" in toml:
failed.append("filerWrite=true: security.toml unexpectedly has [grpc.*] (would need cert mounts)")
if "[jwt.filer_signing]" in toml and "[grpc" not in toml:
print("filerWrite=true: security.toml has [jwt.filer_signing], no [grpc.*]")
# Case 3: filer + admin pods must MOUNT the security ConfigMap
# under filerWrite=true so the JWT key reaches both processes.
# Cert volumes must NOT be present (no mTLS).
for wl in ("test-seaweedfs-filer", "test-seaweedfs-admin"):
vols, mounts = workload_mounts(out, wl)
if vols is None:
failed.append(f"filerWrite=true: workload {wl} not found")
continue
if "security-config" not in vols or "security-config" not in mounts:
failed.append(f"filerWrite=true: {wl} does not mount security-config (IAM gRPC would still fail)")
else:
print(f"filerWrite=true: {wl} mounts security-config")
cert_vols = {v for v in vols if v.endswith("-cert")}
if cert_vols:
failed.append(f"filerWrite=true: {wl} unexpectedly has cert volumes {sorted(cert_vols)}")
# Case 4: enableSecurity=true must still render the full toml
# with both [jwt.signing] and [grpc.*]. Guards against the
# decoupling change accidentally regressing the mTLS path.
out = render({"global.seaweedfs.enableSecurity": "true"})
cm = configmap(out, "test-seaweedfs-security-config")
if cm is None:
failed.append("enableSecurity=true: security ConfigMap missing")
else:
toml = cm["data"]["security.toml"]
missing = [s for s in ("[jwt.signing]", "[grpc.master]") if s not in toml]
if missing:
failed.append(f"enableSecurity=true: security.toml missing {missing}")
else:
print("enableSecurity=true: security.toml has [jwt.signing] + [grpc.*] preserved")
# Case 5: helper must tolerate explicit nulls (gemini-code-assist
# PR review). securityConfig=null was the parens-pattern crash
# the helper review caught.
for null_path in ("global.seaweedfs.securityConfig",
"global.seaweedfs.securityConfig.jwtSigning"):
try:
out = render({null_path: "null"})
except subprocess.CalledProcessError as e:
failed.append(f"{null_path}=null: render failed: {e.output[:200] if e.output else e}")
continue
if configmap(out, "test-seaweedfs-security-config") is not None:
failed.append(f"{null_path}=null: should not render configmap")
else:
print(f"{null_path}=null: render tolerates explicit null")
if failed:
print("\nFAIL:", file=sys.stderr)
for f in failed:
print(f" - {f}", file=sys.stderr)
sys.exit(1)
PYEOF
echo "IAM gRPC decoupling tests passed"
echo "=== Testing with monitoring enabled ==="
helm template test $CHART_DIR \
--set global.seaweedfs.monitoring.enabled=true \
--set global.seaweedfs.monitoring.gatewayHost=prometheus \
--set global.seaweedfs.monitoring.gatewayPort=9091 > /tmp/monitoring.yaml
echo "Monitoring configuration renders correctly"
echo "=== Testing with PVC storage ==="
helm template test $CHART_DIR \
--set master.data.type=persistentVolumeClaim \
--set master.data.size=10Gi \
--set master.data.storageClass=standard > /tmp/pvc.yaml
grep -q "PersistentVolumeClaim" /tmp/pvc.yaml
echo "PVC configuration renders correctly"
echo "=== Testing with custom replicas ==="
helm template test $CHART_DIR \
--set master.replicas=3 \
--set filer.replicas=2 \
--set volume.replicas=3 > /tmp/replicas.yaml
echo "Custom replicas configuration renders correctly"
echo "=== Testing filer with S3 gateway ==="
helm template test $CHART_DIR \
--set filer.s3.enabled=true \
--set filer.s3.enableAuth=true > /tmp/filer-s3.yaml
echo "Filer S3 gateway renders correctly"
echo "=== Testing SFTP enabled ==="
helm template test $CHART_DIR --set sftp.enabled=true > /tmp/sftp.yaml
grep -q "seaweedfs-sftp" /tmp/sftp.yaml
echo "SFTP deployment renders correctly"
echo ""
echo "=== Testing SFTP host key generation ==="
# The chart must not ship host key material: keys are generated
# per install, land in the secret the deployments mount at
# sftp.hostKeysFolder, and must be PKCS#8 ed25519 private keys,
# the shape the server's host key loader parses; it fails to
# start otherwise.
pip install pyyaml -q
python3 - "$CHART_DIR" <<'PYEOF'
import base64, re, subprocess, sys, yaml
chart = sys.argv[1]
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def docs(manifest):
return [d for d in yaml.safe_load_all(manifest) if d]
def secret(manifest, name):
for d in docs(manifest):
if d.get("kind") == "Secret" and d["metadata"]["name"] == name:
return d
return None
def pod_spec(manifest, name):
for d in docs(manifest):
if d.get("kind") in ("Deployment", "StatefulSet") and d["metadata"]["name"] == name:
return d["spec"]["template"]["spec"]
return None
def script_of(spec):
for c in spec["containers"]:
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh":
return cmd[2]
raise AssertionError("no shell command block found")
def volume_secret(spec, volname):
for v in spec.get("volumes", []):
if v["name"] == volname:
return v["secret"]["secretName"]
return None
def mount_path(spec, volname):
for c in spec["containers"]:
for vm in c.get("volumeMounts", []):
if vm["name"] == volname:
return vm["mountPath"]
return None
def parse_ed25519(pem):
m = re.match(r"-----BEGIN PRIVATE KEY-----\n(.+?)-----END PRIVATE KEY-----", pem.strip(), re.S)
if not m:
raise AssertionError("not a PKCS#8 PEM private key")
der = base64.b64decode(m.group(1))
# RFC 8410: fixed PKCS#8 prefix, then the 32-byte seed
prefix = bytes.fromhex("302e020100300506032b657004220420")
if len(der) != 48 or not der.startswith(prefix):
raise AssertionError("not an ed25519 PKCS#8 key")
failed = []
# public-key material of the key the chart used to bundle
BUNDLED = "H4McwcDphteXVullu6q7ephEN1N60z"
out1 = render({"sftp.enabled": "true"})
out2 = render({"sftp.enabled": "true"})
for label, out in (("first", out1), ("second", out2)):
if BUNDLED in out:
failed.append(f"{label} render still contains the formerly bundled host key")
def folder_key(out):
s = secret(out, "test-seaweedfs-sftp-ssh-secret")
if s is None:
return None
return base64.b64decode(s["data"]["ssh_host_ed25519_key"]).decode()
k1, k2 = folder_key(out1), folder_key(out2)
if k1 is None or k2 is None:
failed.append("sftp-ssh-secret not rendered with sftp.enabled=true")
else:
try:
parse_ed25519(k1)
print("generated host key parses as ed25519")
except Exception as e:
failed.append(f"generated host key does not parse: {e}")
if k1 == k2:
failed.append("two renders produced the same host key (key is not generated per install)")
else:
print("host key differs between installs")
legacy1 = secret(out1, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"]
legacy2 = secret(out2, "test-seaweedfs-sftp-secret")["stringData"]["seaweedfs_sftp_ssh_private_key"]
try:
parse_ed25519(legacy1)
except Exception as e:
failed.append(f"sftp-secret ssh key does not parse: {e}")
if legacy1 == legacy2:
failed.append("sftp-secret ssh key identical across renders")
else:
print("sftp-secret ssh key is generated per install")
spec = pod_spec(out1, "test-seaweedfs-sftp")
script = script_of(spec)
if "-sshPrivateKey" in script:
failed.append("sftp deployment passes -sshPrivateKey by default; the file only exists "
"when enableAuth mounts /etc/sw and a missing key file is fatal")
if "-hostKeysFolder=/etc/sw/ssh" not in script:
failed.append("sftp deployment missing -hostKeysFolder=/etc/sw/ssh")
if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret":
failed.append("sftp config-ssh volume does not reference the generated secret")
else:
print("sftp deployment mounts the generated secret at the host keys folder")
out = render({"sftp.enabled": "true", "sftp.existingSshConfigSecret": "my-keys"})
if secret(out, "test-seaweedfs-sftp-ssh-secret") is not None:
failed.append("existingSshConfigSecret set but the default ssh secret still renders")
if volume_secret(pod_spec(out, "test-seaweedfs-sftp"), "config-ssh") != "my-keys":
failed.append("existingSshConfigSecret is not the config-ssh volume source")
else:
print("existingSshConfigSecret replaces the generated secret")
out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true"})
spec = pod_spec(out, "test-seaweedfs-all-in-one")
if secret(out, "test-seaweedfs-sftp-ssh-secret") is None:
failed.append("all-in-one: ssh secret not rendered")
if "-sftp.hostKeysFolder=/etc/sw/ssh" not in script_of(spec):
failed.append("all-in-one: missing -sftp.hostKeysFolder=/etc/sw/ssh")
if volume_secret(spec, "config-ssh") != "test-seaweedfs-sftp-ssh-secret":
failed.append("all-in-one: config-ssh volume does not reference the generated secret")
else:
print("all-in-one mounts the generated secret")
out = render({"sftp.enabled": "true", "sftp.hostKeysFolder": "/keys"})
spec = pod_spec(out, "test-seaweedfs-sftp")
if "-hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys":
failed.append("custom hostKeysFolder: flag and secret mount do not agree")
else:
print("custom hostKeysFolder keeps flag and mount aligned")
out = render({"allInOne.enabled": "true", "allInOne.sftp.enabled": "true",
"allInOne.sftp.hostKeysFolder": "/keys"})
spec = pod_spec(out, "test-seaweedfs-all-in-one")
if "-sftp.hostKeysFolder=/keys" not in script_of(spec) or mount_path(spec, "config-ssh") != "/keys":
failed.append("all-in-one custom hostKeysFolder: flag and secret mount do not agree")
else:
print("all-in-one custom hostKeysFolder keeps flag and mount aligned")
if failed:
print("\nFAIL:", file=sys.stderr)
for f in failed:
print(f" - {f}", file=sys.stderr)
sys.exit(1)
PYEOF
echo "SFTP host key generation tests passed"
echo "=== Testing ingress configurations ==="
helm template test $CHART_DIR \
--set master.ingress.enabled=true \
--set filer.ingress.enabled=true \
--set s3.enabled=true \
--set s3.ingress.enabled=true > /tmp/ingress.yaml
grep -q "kind: Ingress" /tmp/ingress.yaml
echo "Ingress configurations render correctly"
echo "=== Testing COSI driver ==="
helm template test $CHART_DIR --set cosi.enabled=true > /tmp/cosi.yaml
grep -q "seaweedfs-cosi" /tmp/cosi.yaml
echo "COSI driver renders correctly"
echo ""
echo "=== Testing long release name: service names match DNS references ==="
# Use a release name that, combined with chart name "seaweedfs", exceeds 63 chars.
# fullname = "my-very-long-release-name-that-will-cause-truncation-seaweedfs" (65 chars before trunc)
LONG_RELEASE="my-very-long-release-name-that-will-cause-truncation"
# --- Normal mode: master + filer-client services vs helper-produced addresses ---
helm template "$LONG_RELEASE" $CHART_DIR \
--set s3.enabled=true \
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname.yaml
# Extract Service names from metadata
MASTER_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
| grep -- '-master$')
FILER_CLIENT_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname.yaml \
| grep -- '-filer-client$')
# Extract the hostname from WEED_CLUSTER_SW_MASTER in post-install-bucket-hook
MASTER_ADDR=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
FILER_ADDR=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/')
# Extract the hostname from S3 deployment -filer= argument
S3_FILER_HOST=$(grep '\-filer=' /tmp/longname.yaml \
| head -1 | sed 's/.*-filer=\([^:]*\).*/\1/')
# The address helpers produce "<svc>.<namespace>:<port>"; extract just the svc name
MASTER_ADDR_SVC=$(echo "$MASTER_ADDR" | cut -d. -f1)
FILER_ADDR_SVC=$(echo "$FILER_ADDR" | cut -d. -f1)
S3_FILER_SVC=$(echo "$S3_FILER_HOST" | cut -d. -f1)
echo " master Service.name: $MASTER_SVC"
echo " cluster.masterAddress svc: $MASTER_ADDR_SVC"
echo " filer-client Service.name: $FILER_CLIENT_SVC"
echo " cluster.filerAddress svc: $FILER_ADDR_SVC"
echo " S3 -filer= svc: $S3_FILER_SVC"
[ "$MASTER_SVC" = "$MASTER_ADDR_SVC" ] || { echo "FAIL: master service name mismatch"; exit 1; }
[ "$FILER_CLIENT_SVC" = "$FILER_ADDR_SVC" ] || { echo "FAIL: filer-client service name mismatch"; exit 1; }
[ "$FILER_CLIENT_SVC" = "$S3_FILER_SVC" ] || { echo "FAIL: S3 -filer= does not match filer-client service"; exit 1; }
echo "Normal mode: service names match DNS references with long release name"
# --- All-in-one mode: all-in-one service vs both helper addresses ---
helm template "$LONG_RELEASE" $CHART_DIR \
--set allInOne.enabled=true \
--set global.seaweedfs.createBuckets[0].name=test > /tmp/longname-aio.yaml
AIO_SVC=$(awk '/kind: Service/{found=1} found && /^ *name:/{print $2; found=0}' /tmp/longname-aio.yaml \
| grep -- '-all-in-one$')
AIO_MASTER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_MASTER' -A1 /tmp/longname-aio.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
AIO_FILER_ADDR_SVC=$(grep 'WEED_CLUSTER_SW_FILER' -A1 /tmp/longname-aio.yaml \
| grep 'value:' | head -1 | sed 's/.*value: *"\{0,1\}\([^":]*\).*/\1/' | cut -d. -f1)
echo " all-in-one Service.name: $AIO_SVC"
echo " cluster.masterAddress svc: $AIO_MASTER_ADDR_SVC"
echo " cluster.filerAddress svc: $AIO_FILER_ADDR_SVC"
[ "$AIO_SVC" = "$AIO_MASTER_ADDR_SVC" ] || { echo "FAIL: all-in-one master address mismatch"; exit 1; }
[ "$AIO_SVC" = "$AIO_FILER_ADDR_SVC" ] || { echo "FAIL: all-in-one filer address mismatch"; exit 1; }
echo "All-in-one mode: service names match DNS references with long release name"
echo ""
echo "=== Testing security+S3: no blank lines in shell command blocks ==="
# Render the three manifests that include seaweedfs.s3.tlsArgs:
# filer-statefulset, s3-deployment, all-in-one-deployment
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set filer.s3.enabled=true \
--set s3.enabled=true > /tmp/security-s3.yaml
helm template test $CHART_DIR \
--set global.seaweedfs.enableSecurity=true \
--set allInOne.enabled=true \
--set allInOne.s3.enabled=true > /tmp/security-aio.yaml
pip install pyyaml -q
python3 - /tmp/security-s3.yaml /tmp/security-aio.yaml <<'PYEOF'
import yaml, sys
errors = []
for path in sys.argv[1:]:
with open(path) as f:
docs = list(yaml.safe_load_all(f))
for doc in docs:
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
continue
name = doc["metadata"]["name"]
for c in doc["spec"]["template"]["spec"].get("containers", []):
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
script = cmd[2]
for i, line in enumerate(script.splitlines(), 1):
if line.strip() == "":
errors.append(f"{path}: {name}/{c['name']} has blank line at script line {i}")
if errors:
for e in errors:
print(f"FAIL: {e}", file=sys.stderr)
print("Rendered with: global.seaweedfs.enableSecurity=true, filer.s3.enabled=true, s3.enabled=true, allInOne.enabled=true", file=sys.stderr)
sys.exit(1)
print("No blank lines in security+S3 command blocks")
PYEOF
echo ""
echo "=== Testing security+S3: -cert.file/-key.file gated on httpsPort (issue #9202) ==="
# Regression test: when enableSecurity=true but *.httpsPort is 0 (the default),
# the chart must NOT emit -cert.file / -key.file to the S3 frontend. Passing
# them promotes weed s3's main -port to HTTPS (see weed/command/s3.go), which
# makes the HTTP readinessProbe spam "TLS handshake error ... client sent an
# HTTP request to an HTTPS server" into the pod log.
#
# When *.httpsPort > 0, both -port.https and cert/key args MUST be emitted
# together so the opt-in HTTPS listener actually has credentials.
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def script_of(manifest, kind_name):
for doc in yaml.safe_load_all(manifest):
if not doc or doc.get("kind") not in ("Deployment", "StatefulSet"):
continue
if doc["metadata"]["name"] != kind_name:
continue
for c in doc["spec"]["template"]["spec"]["containers"]:
cmd = c.get("command", [])
if len(cmd) >= 3 and cmd[0] == "/bin/sh" and cmd[1] == "-ec":
return cmd[2]
raise AssertionError(f"no container script for {kind_name}")
cases = [
# (values, workload-name, httpsPort-set?, arg-prefix)
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true"},
"test-seaweedfs-s3", False, ""),
({"global.seaweedfs.enableSecurity": "true",
"s3.enabled": "true",
"s3.httpsPort": "8443"},
"test-seaweedfs-s3", True, ""),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true"},
"test-seaweedfs-filer", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"filer.s3.enabled": "true",
"filer.s3.httpsPort": "8444"},
"test-seaweedfs-filer", True, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true"},
"test-seaweedfs-all-in-one", False, "s3."),
({"global.seaweedfs.enableSecurity": "true",
"allInOne.enabled": "true",
"allInOne.s3.enabled": "true",
"allInOne.s3.httpsPort": "8445"},
"test-seaweedfs-all-in-one", True, "s3."),
]
failed = False
for values, name, https_on, prefix in cases:
script = script_of(render(values), name)
cert_flag = f"-{prefix}cert.file="
key_flag = f"-{prefix}key.file="
https_flag = f"-{prefix}port.https="
has_cert = cert_flag in script
has_key = key_flag in script
has_https = https_flag in script
label = f"{name} (httpsPort {'set' if https_on else 'unset'})"
if https_on:
if not (has_cert and has_key and has_https):
print(f"FAIL: {label}: expected {cert_flag}, {key_flag}, {https_flag} all present "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"{label}: cert/key/https args emitted together")
else:
if has_cert or has_key or has_https:
print(f"FAIL: {label}: expected none of {cert_flag}/{key_flag}/{https_flag}; "
f"main S3 -port would silently become HTTPS and break HTTP probes "
f"(got cert={has_cert} key={has_key} https={has_https})", file=sys.stderr)
failed = True
else:
print(f"{label}: no TLS args emitted, main -port stays HTTP")
# bash -n: pin down that the rendered script parses. Guards against
# a future helper change that leaves a dangling `\` with nothing
# after it (every current caller already exits cleanly because
# bash treats trailing `\<newline><EOF>` as line-continuation to
# an empty line — but keep the contract explicit).
parse = subprocess.run(["bash", "-n"], input=script, text=True,
capture_output=True)
if parse.returncode != 0:
print(f"FAIL: {label}: bash -n rejected rendered script: {parse.stderr.strip()}",
file=sys.stderr)
failed = True
sys.exit(1 if failed else 0)
PYEOF
echo ""
echo "=== Testing all-in-one env: a key in both global and component renders once ==="
# Regression: all-in-one looped global and component extraEnvironmentVars
# in two separate ranges, emitting duplicate env entries for any key set
# in both maps. Render a shared key and assert it appears exactly once in
# the all-in-one container, with the component value winning (consistent
# with the merge helper the other components already use). pyyaml is
# installed by the earlier IAM gRPC block in this same step.
helm template test $CHART_DIR \
--set allInOne.enabled=true \
--set global.seaweedfs.extraEnvironmentVars.WEED_SHARED=fromGlobal \
--set allInOne.extraEnvironmentVars.WEED_SHARED=fromComponent > /tmp/aio-env.yaml
python3 - /tmp/aio-env.yaml <<'PYEOF'
import sys, yaml
from collections import Counter
docs = [d for d in yaml.safe_load_all(open(sys.argv[1])) if d]
dep = next(d for d in docs if d.get("kind") == "Deployment"
and d["metadata"]["name"].endswith("all-in-one"))
envs = [e["name"] for c in dep["spec"]["template"]["spec"]["containers"]
for e in c.get("env", [])]
dups = {k: v for k, v in Counter(envs).items() if v > 1}
if dups:
print(f"FAIL: duplicate env entries in all-in-one container: {dups}", file=sys.stderr)
sys.exit(1)
val = next(e.get("value") for c in dep["spec"]["template"]["spec"]["containers"]
for e in c.get("env", []) if e["name"] == "WEED_SHARED")
if val != "fromComponent":
print(f"FAIL: WEED_SHARED should take the component value 'fromComponent', got '{val}'",
file=sys.stderr)
sys.exit(1)
print("all-in-one env: shared key renders once, component value wins")
PYEOF
echo "=== Testing bucket versioning: YAML bool false suspends like string \"false\" ==="
# bool false used to be a silent no-op while string "false" suspended.
BOOL_FALSE=$(helm template test $CHART_DIR \
--set s3.enabled=true \
--set s3.createBuckets[0].name=verbucket \
--set s3.createBuckets[0].versioning=false | grep 's3.bucket.versioning -name verbucket' || true)
echo "$BOOL_FALSE" | grep -q -- '-status Suspended' || { echo "FAIL: bool false versioning did not Suspend the bucket"; exit 1; }
echo "Bucket versioning: YAML bool false suspends consistently with string \"false\""
echo ""
echo "=== Testing hook Job labels ==="
# The hook Jobs were the only pods in the chart without the standard
# app.kubernetes.io label set, so nothing label-based could target
# them - NetworkPolicy podSelectors, monitoring, kubectl -l. Assert
# both the Job and its pod template carry the same name/instance/
# component triple the other components use, and that the triple is
# the hook's own so a per-component selector cannot match it too.
#
# Only the bucket hook is covered: the volume resize hook is gated on
# lookup finding a StatefulSet with a smaller PVC than requested, and
# lookup returns nothing under helm template, so it never renders here.
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
# The three a selector keys on, and the full set the other workloads
# carry - a missing chart or managed-by label is not a selector
# problem, but it does leave the hook Jobs looking unlike everything
# else the release owns.
TRIPLE = ("app.kubernetes.io/name", "app.kubernetes.io/instance",
"app.kubernetes.io/component")
# The labels that identify the release rather than the workload, and
# so have to hold the same values everywhere. They are compared
# against a workload that already renders them instead of being
# spelled out here, which keeps the chart version out of the test.
# managed-by is not among them on purpose: the chart puts it on
# workload metadata but not on pod templates, so it cannot be part of
# a cross-workload comparison. Its presence is still checked below.
RELEASE = ("app.kubernetes.io/name", "app.kubernetes.io/instance",
"helm.sh/chart")
STANDARD = TRIPLE + ("helm.sh/chart", "app.kubernetes.io/managed-by")
def render(values):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
return subprocess.check_output(args, text=True)
def docs(manifest):
return [d for d in yaml.safe_load_all(manifest) if d]
def hook_job(manifest):
for d in docs(manifest):
if d.get("kind") == "Job" and d["metadata"]["name"].endswith("-bucket-hook"):
return d
return None
def triple(labels):
return {k: labels.get(k) for k in TRIPLE}
def release(labels):
return {k: labels.get(k) for k in RELEASE}
def reference(manifest):
"""Any long-running workload; they all carry the release labels."""
for d in docs(manifest):
if d.get("kind") in ("Deployment", "StatefulSet"):
return d
return None
modes = {
"s3": {"s3.enabled": "true",
"s3.createBuckets[0].name": "b"},
"filer.s3": {"filer.s3.enabled": "true",
"filer.s3.createBuckets[0].name": "b"},
"allInOne": {"allInOne.enabled": "true",
"allInOne.s3.enabled": "true",
"allInOne.s3.createBuckets[0].name": "b"},
}
failed = []
for mode, values in modes.items():
before = len(failed)
out = render(values)
job = hook_job(out)
if job is None:
failed.append(f"{mode}: bucket hook Job not rendered")
continue
job_labels = job["metadata"].get("labels", {})
pod_labels = job["spec"]["template"]["metadata"].get("labels", {})
ref = reference(out)
if ref is None:
failed.append(f"{mode}: no workload to compare the release labels against")
continue
ref_labels = release(ref["spec"]["template"]["metadata"].get("labels", {}))
for where, labels in (("Job", job_labels), ("pod", pod_labels)):
missing = [k for k in STANDARD if not labels.get(k)]
if missing:
failed.append(f"{mode}: bucket hook {where} has no {missing}, "
"nothing can select it")
component = labels.get("app.kubernetes.io/component")
if component != "bucket-hook":
failed.append(f"{mode}: bucket hook {where} component is "
f"{component!r}, expected 'bucket-hook'")
# Present is not enough: the values have to be the release's
# own, or a selector written for this release misses the hook.
if release(labels) != ref_labels:
failed.append(f"{mode}: bucket hook {where} release labels "
f"{release(labels)} differ from "
f"{ref['metadata']['name']}'s {ref_labels}")
# A selector written against the Job has to find its pods.
if triple(job_labels) != triple(pod_labels):
failed.append(f"{mode}: bucket hook Job and pod disagree: "
f"{triple(job_labels)} vs {triple(pod_labels)}")
# The triple must not also match another component's pods, or a
# selector meant for that component would pull the hook pod in.
for d in docs(out):
if d.get("kind") not in ("Deployment", "StatefulSet"):
continue
other = d["spec"]["template"]["metadata"].get("labels", {})
if triple(other) == triple(pod_labels):
failed.append(f"{mode}: bucket hook pod shares its label triple "
f"with {d['metadata']['name']}")
if len(failed) == before:
print(f"{mode}: bucket hook Job and pod carry a distinct label triple")
if failed:
print("\nFAIL:", file=sys.stderr)
for f in failed:
print(f" - {f}", file=sys.stderr)
sys.exit(1)
PYEOF
echo "Hook Job label tests passed"
echo ""
echo "=== Testing NetworkPolicy rendering ==="
# The policies are only exercised for real by the networkpolicy-install
# job below. These assertions cover what template rendering can see:
# that the flag stays off by default, that every deployed component has
# exactly one policy, that each policy allows every port its workload
# declares, and that egress stays a separate opt-in.
python3 - "$CHART_DIR" <<'PYEOF'
import subprocess, sys, yaml
chart = sys.argv[1]
def render(values, json_values=None):
args = ["helm", "template", "test", chart]
for k, v in values.items():
args += ["--set", f"{k}={v}"]
for k, v in (json_values or {}).items():
args += ["--set-json", f"{k}={v}"]
return subprocess.check_output(args, text=True, stderr=subprocess.STDOUT)
def expect_failure(values, needle, label, json_values=None):
try:
render(values, json_values)
except subprocess.CalledProcessError as e:
if needle not in (e.output or ""):
failed.append(f"{label}: unexpected error: {(e.output or '')[:200]}")
else:
print(label)
return
failed.append(f"{label}: render should have failed")
def docs(manifest):
return [d for d in yaml.safe_load_all(manifest) if d]
def component(labels):
return labels.get("app.kubernetes.io/component")
def policies(manifest):
out = {}
for d in docs(manifest):
if d.get("kind") != "NetworkPolicy":
continue
out[component(d["spec"]["podSelector"]["matchLabels"])] = d
return out
def workloads(manifest, kinds=("Deployment", "StatefulSet")):
out = {}
for d in docs(manifest):
if d.get("kind") in kinds:
out[component(d["spec"]["template"]["metadata"]["labels"])] = d
return out
def allowed_ports(policy):
ports = set()
for rule in policy["spec"].get("ingress") or []:
for p in rule.get("ports") or []:
ports.add(p["port"])
return ports
def container_ports(workload):
ports = set()
for c in workload["spec"]["template"]["spec"].get("containers", []):
for p in c.get("ports") or []:
ports.add(p["containerPort"])
return ports
EVERYTHING = {
"s3.enabled": "true",
"sftp.enabled": "true",
"admin.enabled": "true",
"worker.enabled": "true",
"cosi.enabled": "true",
"s3.createBuckets[0].name": "b",
"volumes.ssd.port": "8081",
"global.seaweedfs.monitoring.enabled": "true",
}
failed = []
# Off by default: the chart has never shipped a NetworkPolicy and must not
# start now, or every existing release in a default-deny namespace changes
# behaviour on the next upgrade.
if policies(render(EVERYTHING)):
failed.append("networkPolicy.enabled unset: policies rendered anyway")
else:
print("networkPolicy off by default: no policies rendered")
on = dict(EVERYTHING, **{"networkPolicy.enabled": "true"})
out = render(on)
pols = policies(out)
wls = workloads(out)
# Every workload gets exactly one policy, and every policy has a workload.
# A component with no policy is wide open under default-deny; a policy with
# no component is dead weight that hides a renamed label.
for comp in wls:
if comp not in pols:
failed.append(f"{comp}: workload has no NetworkPolicy")
for comp in pols:
# The hook Jobs are covered too, and the resize hook Job only renders
# when a cluster lookup says a PVC needs growing, so it is never in the
# rendered set here.
if comp not in wls and comp not in ("bucket-hook", "volume-resize-hook"):
failed.append(f"{comp}: NetworkPolicy selects a component that is not deployed")
# The ports a component listens on come from the same values as its
# containerPorts, so the two must agree. This is what catches a port added to
# a workload and forgotten in the policy - the failure mode that only shows up
# once someone turns the flag on.
for comp, wl in wls.items():
if comp not in pols:
continue
declared = container_ports(wl)
allowed = allowed_ports(pols[comp])
missing = sorted(declared - allowed)
if missing:
failed.append(f"{comp}: listens on {missing} but its policy does not allow it")
if not failed:
print("every workload has a policy covering all of its containerPorts")
# Egress is its own opt-in: with it off the policies must not constrain
# outbound traffic at all, or enabling networkPolicy alone would cut the filer
# off from its store.
for comp, p in pols.items():
if "Egress" in p["spec"]["policyTypes"]:
failed.append(f"{comp}: Egress in policyTypes while networkPolicy.egress.enabled is false")
if not any("Egress" in p["spec"]["policyTypes"] for p in pols.values()):
print("egress off by default: policies are ingress-only")
# Components that need the API server must not silently lose it: rendering
# fails with a pointer to the value instead.
expect_failure(dict(on, **{"networkPolicy.egress.enabled": "true"}),
"kubeApiServer.cidrs is empty",
"empty kubeApiServer.cidrs fails the render with a pointer to the value")
egress_on = dict(on, **{
"networkPolicy.egress.enabled": "true",
"networkPolicy.egress.kubeApiServer.cidrs[0]": "10.96.0.1/32",
})
# An empty port or peer list is "everything" in a NetworkPolicy, not
# "nothing", so the two places that could be emptied have to be refused
# rather than quietly widened.
expect_failure(egress_on, "kubeApiServer.ports is empty",
"empty kubeApiServer.ports is refused instead of allowing every port",
json_values={"networkPolicy.egress.kubeApiServer.ports": "[]"})
expect_failure(egress_on, "allowDNS is on but both",
"DNS with no selector is refused instead of allowing every pod",
json_values={"networkPolicy.egress.dnsNamespaceSelector": "null",
"networkPolicy.egress.dnsPodSelector": "null"})
# Dropping one of the two DNS selectors must leave the key out, not
# render it as null - null namespaceSelector means this namespace and
# null podSelector means every pod in the peer namespace.
for dropped in ("dnsNamespaceSelector", "dnsPodSelector"):
out = render(egress_on, {f"networkPolicy.egress.{dropped}": "null"})
nulls = [(c, k) for c, p in policies(out).items()
for r in p["spec"]["egress"]
for t in r.get("to") or []
for k, v in t.items() if v is None]
nulls += [(c, "ports") for c, p in policies(out).items()
for r in p["spec"]["egress"] if "ports" in r and r["ports"] is None]
if nulls:
failed.append(f"{dropped}=null: rendered null keys {sorted(set(nulls))}")
else:
print(f"{dropped}=null: the key is left out rather than rendered as null")
out = render(egress_on)
pols = policies(out)
# DNS for everyone: every component addresses its peers by service name.
for comp, p in pols.items():
dns = [r for r in p["spec"]["egress"]
if {x["port"] for x in r.get("ports") or []} == {53}]
if not dns:
failed.append(f"{comp}: egress on but no DNS rule")
if not failed:
print("every policy allows DNS when egress is on")
# The API server rule goes only to the components that talk to it.
apiserver = {c for c, p in pols.items()
if any("ipBlock" in t for r in p["spec"]["egress"] for t in r.get("to") or [])}
expected = {"admin", "objectstorage-provisioner", "volume-resize-hook"}
if apiserver != expected:
failed.append(f"API server egress granted to {sorted(apiserver)}, expected {sorted(expected)}")
else:
print(f"API server egress limited to {sorted(expected)}")
# The resize hook runs as a pre-install hook at weight 0, before the release
# manifest is applied, so its policy has to be a hook itself and has to sort
# ahead of the Job.
rh = pols["volume-resize-hook"]["metadata"].get("annotations", {})
if rh.get("helm.sh/hook") != "pre-install,pre-upgrade":
failed.append(f"volume-resize-hook policy is not a pre-install hook: {rh}")
elif int(rh.get("helm.sh/hook-weight", 0)) >= 0:
failed.append(f"volume-resize-hook policy weight {rh.get('helm.sh/hook-weight')} does not sort before the Job at 0")
else:
print("volume-resize-hook policy is a pre-install hook ahead of the Job")
# The bucket hook is post-install, so the release manifest is already applied;
# its policy must be a plain resource that uninstall cleans up.
if "helm.sh/hook" in (pols["bucket-hook"]["metadata"].get("annotations") or {}):
failed.append("bucket-hook policy is a hook resource; post-install runs after the manifest is applied")
else:
print("bucket-hook policy is a plain release resource")
if failed:
print("\nFAIL:", file=sys.stderr)
for f in failed:
print(f" - {f}", file=sys.stderr)
sys.exit(1)
PYEOF
echo "NetworkPolicy rendering tests passed"
echo "All template rendering tests passed!"
- name: Create kind cluster
uses: helm/kind-action@v1.14.0
- name: Run chart-testing (install)
run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --chart-dirs k8s/charts
- name: Verify SFTP host key secret lifecycle
run: |
set -e
CHART_DIR="k8s/charts/seaweedfs"
NS="sftp-hostkey"
SECRET="hk-seaweedfs-sftp-ssh-secret"
SFTP_ARGS="--set sftp.enabled=true --set master.enabled=false --set volume.enabled=false --set filer.enabled=false"
kubectl create namespace "$NS"
echo "=== install generates a host key, upgrade keeps it ==="
helm install hk $CHART_DIR -n "$NS" $SFTP_ARGS
KEY1=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}')
[ -n "$KEY1" ] || { echo "FAIL: install did not create a host key"; exit 1; }
echo "$KEY1" | base64 -d | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: host key is not a PEM private key"; exit 1; }
helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
KEY2=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}')
[ "$KEY1" = "$KEY2" ] || { echo "FAIL: host key changed across upgrade"; exit 1; }
echo "host key survives upgrade"
echo "=== the key the chart used to bundle is replaced ==="
kubectl delete secret "$SECRET" -n "$NS"
kubectl create secret generic "$SECRET" -n "$NS" \
--from-literal=ssh_host_ed25519_key="stand-in H4McwcDphteXVullu6q7ephEN1N60z stand-in"
helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
ROTATED=$(kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.ssh_host_ed25519_key}' | base64 -d)
case "$ROTATED" in
*H4McwcDphteXVullu6q7ephEN1N60z*) echo "FAIL: bundled key survived the upgrade"; exit 1;;
esac
echo "$ROTATED" | grep -q "BEGIN PRIVATE KEY" || { echo "FAIL: replacement is not a generated key"; exit 1; }
echo "bundled key rotated to a generated one"
echo "=== operator-managed keys are kept as-is ==="
kubectl delete secret "$SECRET" -n "$NS"
ssh-keygen -q -t ed25519 -N "" -C "" -f /tmp/operator_key
kubectl create secret generic "$SECRET" -n "$NS" --from-file=my_key=/tmp/operator_key
helm upgrade hk $CHART_DIR -n "$NS" $SFTP_ARGS
kubectl get secret "$SECRET" -n "$NS" -o jsonpath='{.data.my_key}' | base64 -d | cmp -s - /tmp/operator_key \
|| { echo "FAIL: operator key was modified"; exit 1; }
NKEYS=$(kubectl get secret "$SECRET" -n "$NS" -o json | jq '.data | length')
[ "$NKEYS" = "1" ] || { echo "FAIL: expected only the operator key, found $NKEYS entries"; exit 1; }
echo "operator key kept, no extra key generated"
kubectl delete namespace "$NS"
echo "SFTP host key lifecycle tests passed"
- name: Verify install into a default-deny namespace
run: |
set -e
CHART_DIR="k8s/charts/seaweedfs"
NS="netpol"
# kind enforces NetworkPolicy out of the box since v0.24 (kindnetd
# runs sigs.k8s.io/kube-network-policies), so the cluster created for
# chart-testing above is enough and no extra CNI is needed. The two
# probes at the end fail loudly if that ever stops being true, rather
# than letting this pass vacuously.
kubectl create namespace "$NS"
kubectl apply -n "$NS" -f - <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: default-deny
spec:
podSelector: {}
policyTypes: [Ingress, Egress]
EOF
# The backend address, not the kubernetes service ClusterIP: kube-proxy
# rewrites the destination, so an ipBlock has to name the real endpoint.
APISERVER=$(kubectl get endpointslice kubernetes \
-o jsonpath='{.endpoints[0].addresses[0]}' 2>/dev/null || true)
if [ -z "$APISERVER" ]; then
APISERVER=$(kubectl get endpoints kubernetes -o jsonpath='{.subsets[0].addresses[0].ip}')
fi
echo "kube-apiserver at $APISERVER"
echo "=== install with the policies on ==="
# Without them this hangs: the components cannot resolve or reach each
# other, and the post-install bucket hook waits on master and filer
# until it gives up. --wait covers the components, and helm fails the
# release if the hook Job does not finish, so a clean install is the
# assertion.
helm install np $CHART_DIR -n "$NS" --wait --timeout 8m \
--set s3.enabled=true \
--set s3.createBuckets[0].name=testbucket \
--set networkPolicy.enabled=true \
--set networkPolicy.egress.enabled=true \
--set networkPolicy.egress.kubeApiServer.cidrs[0]="$APISERVER/32"
echo "release came up and the bucket hook finished under default-deny"
FILER_IP=$(kubectl get pod -n "$NS" -l app.kubernetes.io/component=filer \
-o jsonpath='{.items[0].status.podIP}')
echo "filer pod at $FILER_IP"
# Both probes get their own all-egress policy, so the namespace-wide
# default-deny is not what decides the outcome: the only thing left in
# the way is the filer's own policy, which admits release pods only.
# Probing the pod IP keeps DNS out of it.
kubectl apply -n "$NS" -f - <<'EOF'
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: probe-egress
spec:
podSelector:
matchExpressions:
- key: probe
operator: Exists
policyTypes: [Egress]
egress:
- {}
EOF
probe() {
local name=$1
local labels=$2
kubectl run "$name" -n "$NS" --image=busybox:1.36 --restart=Never \
--labels="probe=$name,$labels" --command -- \
sh -c "wget -T 5 -q -O /dev/null http://$FILER_IP:8888/; echo exit=\$?"
kubectl wait -n "$NS" --for=jsonpath='{.status.phase}'=Succeeded \
"pod/$name" --timeout=120s >/dev/null
kubectl logs -n "$NS" "$name"
}
# A pod carrying the release labels is what the filer's policy allows.
# This has to succeed, otherwise the policies are blocking traffic they
# are supposed to permit - or nothing is enforced and the next probe
# would be meaningless.
ALLOWED=$(probe probe-allowed "app.kubernetes.io/name=seaweedfs,app.kubernetes.io/instance=np")
echo "labelled probe: $ALLOWED"
case "$ALLOWED" in
*exit=0*) echo "a pod with the release labels reaches the filer";;
*) echo "FAIL: the filer policy rejects a pod carrying the release labels"; exit 1;;
esac
# The same probe without those labels must not get through.
DENIED=$(probe probe-denied "role=outsider")
echo "unlabelled probe: $DENIED"
case "$DENIED" in
*exit=0*) echo "FAIL: a pod outside the release reached the filer; the policy over-allows"; exit 1;;
*) echo "the filer policy refuses a pod outside the release";;
esac
kubectl delete namespace "$NS" --wait=false
echo "default-deny namespace tests passed"
Reference in New Issue View Git Blame Copy Permalink