mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-22 04:07:08 +01:00
Compare commits
17
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5ef99c1d8d | ||
|
|
8e15110b4f | ||
|
|
f13b763552 | ||
|
|
df30c883fe | ||
|
|
5a1b0abc20 | ||
|
|
b2dd93eac8 | ||
|
|
5f7fefadae | ||
|
|
c09009a74f | ||
|
|
c0509cf499 | ||
|
|
63806b7073 | ||
|
|
504de9b562 | ||
|
|
d80e905fc7 | ||
|
|
799fd10342 | ||
|
|
a91b333658 | ||
|
|
4f70ad015e | ||
|
|
ddd3b3666f | ||
|
|
cdb01f8150 |
No files matched your search
@@ -67,6 +67,8 @@ Download the latest binary from the [releases](https://github.com/seaweedfs/seaw
|
||||
curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
```
|
||||
|
||||
For SeaweedFS Enterprise binaries, pass `--enterprise` to the script: `... | bash -s -- --enterprise`.
|
||||
|
||||
Then start a ready-to-use S3 object store:
|
||||
|
||||
```bash
|
||||
|
||||
+65
-26
@@ -7,12 +7,14 @@
|
||||
# curl -fsSL https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/install.sh | bash
|
||||
# curl -fsSL ... | bash -s -- --component volume-rust --large-disk
|
||||
# curl -fsSL ... | bash -s -- --version 4.34 --dir /usr/local/bin
|
||||
# curl -fsSL ... | bash -s -- --enterprise --component all
|
||||
#
|
||||
# Options:
|
||||
# --component COMP Which binary to install: weed, volume-rust, worker-rust, vector, all (default: weed)
|
||||
# --version VER Release version tag (default: latest)
|
||||
# --large-disk Use large disk variant (5-byte offset, 8TB max volume)
|
||||
# --dir DIR Installation directory (default: /usr/local/bin)
|
||||
# --enterprise Install SeaweedFS Enterprise binaries (from seaweedfs/artifactory)
|
||||
# --help Show this help message
|
||||
|
||||
set -euo pipefail
|
||||
@@ -24,6 +26,10 @@ LARGE_DISK=false
|
||||
INSTALL_DIR="/usr/local/bin"
|
||||
WORKER_INSTALLED=false
|
||||
VECTOR_INSTALLED=false
|
||||
VOLUME_INSTALLED=false
|
||||
ENTERPRISE=false
|
||||
# Enterprise assets carry this after the binary name, e.g. weed-vector-enterprise_linux_amd64.tar.gz.
|
||||
EDITION=""
|
||||
|
||||
# Colors (if terminal supports them)
|
||||
if [ -t 1 ]; then
|
||||
@@ -53,6 +59,7 @@ while [ $# -gt 0 ]; do
|
||||
--version) VERSION="$2"; shift 2 ;;
|
||||
--large-disk) LARGE_DISK=true; shift ;;
|
||||
--dir) INSTALL_DIR="$2"; shift 2 ;;
|
||||
--enterprise) ENTERPRISE=true; REPO="seaweedfs/artifactory"; EDITION="-enterprise"; shift ;;
|
||||
--help|-h) usage ;;
|
||||
*) error "Unknown option: $1. Use --help for usage." ;;
|
||||
esac
|
||||
@@ -81,15 +88,22 @@ detect_platform() {
|
||||
}
|
||||
|
||||
# Get latest release tag from GitHub API
|
||||
# The Enterprise repo also releases other products (vfs-*, previews), which can
|
||||
# be its latest release, so take the newest plain version tag like 4.53.
|
||||
get_latest_version() {
|
||||
local url="https://api.github.com/repos/${REPO}/releases/latest"
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
url="https://api.github.com/repos/${REPO}/releases?per_page=30"
|
||||
fi
|
||||
local body
|
||||
if command -v curl &>/dev/null; then
|
||||
curl -fsSL "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
body="$(curl -fsSL "$url")"
|
||||
elif command -v wget &>/dev/null; then
|
||||
wget -qO- "$url" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/'
|
||||
body="$(wget -qO- "$url")"
|
||||
else
|
||||
error "Neither curl nor wget found. Please install one."
|
||||
fi
|
||||
echo "$body" | grep '"tag_name"' | sed 's/.*"tag_name": *"\([^"]*\)".*/\1/' | grep -E '^[0-9]+(\.[0-9]+)+$' | head -1 || true
|
||||
}
|
||||
|
||||
# Download a file
|
||||
@@ -110,17 +124,22 @@ go_asset_name() {
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
suffix="${suffix}_large_disk"
|
||||
fi
|
||||
echo "${suffix}.tar.gz"
|
||||
if [ "$ENTERPRISE" = true ]; then
|
||||
suffix="weed-enterprise-${suffix}"
|
||||
fi
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "${suffix}.zip"
|
||||
else
|
||||
echo "${suffix}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
# Build Rust volume server asset name
|
||||
rust_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
local prefix="weed-volume"
|
||||
local prefix="weed-volume${EDITION}"
|
||||
if [ "$LARGE_DISK" = true ]; then
|
||||
prefix="weed-volume_large_disk"
|
||||
else
|
||||
prefix="weed-volume"
|
||||
prefix="${prefix}_large_disk"
|
||||
fi
|
||||
local suffix="${os}_${arch}"
|
||||
if [ "$os" = "windows" ]; then
|
||||
@@ -130,6 +149,19 @@ rust_asset_name() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Enterprise publishes the Rust volume server for linux only.
|
||||
rust_volume_platform_supported() {
|
||||
case "$2" in
|
||||
amd64|arm64) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$1" in
|
||||
linux) return 0 ;;
|
||||
darwin|windows) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Does a release carry this asset? Answers 0 for yes and 1 for a 404, and stops
|
||||
# the installer on anything else: a rate limit or a network blip must not read
|
||||
# as "this release predates the binary" and quietly skip it.
|
||||
@@ -154,22 +186,23 @@ asset_exists() {
|
||||
# maintains tables through the namespace and never opens a volume file.
|
||||
worker_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
echo "weed-worker_${os}_${arch}.tar.gz"
|
||||
echo "weed-worker${EDITION}_${os}_${arch}.tar.gz"
|
||||
}
|
||||
|
||||
# The vector engine behind the S3 Vectors API, built like the worker.
|
||||
vector_asset_name() {
|
||||
local os="$1" arch="$2"
|
||||
if [ "$os" = "windows" ]; then
|
||||
echo "weed-vector_${os}_${arch}.zip"
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.zip"
|
||||
else
|
||||
echo "weed-vector_${os}_${arch}.tar.gz"
|
||||
echo "weed-vector${EDITION}_${os}_${arch}.tar.gz"
|
||||
fi
|
||||
}
|
||||
|
||||
vector_platform_supported() {
|
||||
case "$1/$2" in
|
||||
linux/amd64|linux/arm64|darwin/amd64|darwin/arm64|windows/amd64) return 0 ;;
|
||||
linux/amd64|linux/arm64) return 0 ;;
|
||||
darwin/amd64|darwin/arm64|windows/amd64) [ "$ENTERPRISE" != true ] ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
@@ -189,7 +222,10 @@ install_component() {
|
||||
download "$download_url" "${tmpdir}/${asset_name}"
|
||||
|
||||
info "Extracting ${asset_name}..."
|
||||
tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir"
|
||||
case "$asset_name" in
|
||||
*.zip) unzip -q "${tmpdir}/${asset_name}" -d "$tmpdir" ;;
|
||||
*) tar xzf "${tmpdir}/${asset_name}" -C "$tmpdir" ;;
|
||||
esac
|
||||
|
||||
# The Go release action puts the binary inside a directory
|
||||
local weed_bin
|
||||
@@ -202,20 +238,16 @@ install_component() {
|
||||
fi
|
||||
|
||||
chmod +x "$weed_bin"
|
||||
install_binary "$weed_bin" "weed"
|
||||
ok "Installed weed to ${INSTALL_DIR}/weed"
|
||||
local dest_name="weed"
|
||||
if [ "$os" = "windows" ]; then
|
||||
dest_name="weed.exe"
|
||||
fi
|
||||
install_binary "$weed_bin" "$dest_name"
|
||||
ok "Installed weed to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
volume-rust)
|
||||
# Check platform support for Rust volume server
|
||||
case "$os" in
|
||||
linux|darwin|windows) ;;
|
||||
*) error "Rust volume server is not available for ${os}. Supported: linux, darwin, windows" ;;
|
||||
esac
|
||||
case "$arch" in
|
||||
amd64|arm64) ;;
|
||||
*) error "Rust volume server is not available for ${arch}. Supported: amd64, arm64" ;;
|
||||
esac
|
||||
rust_volume_platform_supported "$os" "$arch" || error "The Rust volume server is not available for ${os}/${arch}"
|
||||
|
||||
asset_name="$(rust_asset_name "$os" "$arch")"
|
||||
download_url="https://github.com/${REPO}/releases/download/${VERSION}/${asset_name}"
|
||||
@@ -247,6 +279,7 @@ install_component() {
|
||||
dest_name="weed-volume.exe"
|
||||
fi
|
||||
install_binary "$rust_bin" "$dest_name"
|
||||
VOLUME_INSTALLED=true
|
||||
ok "Installed weed-volume to ${INSTALL_DIR}/${dest_name}"
|
||||
;;
|
||||
|
||||
@@ -359,7 +392,11 @@ main() {
|
||||
case "$COMPONENT" in
|
||||
all)
|
||||
install_component "weed" "$os" "$arch"
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
if rust_volume_platform_supported "$os" "$arch"; then
|
||||
install_component "volume-rust" "$os" "$arch"
|
||||
else
|
||||
warn "Skipping the Rust volume server: no build for ${os}/${arch}"
|
||||
fi
|
||||
# The worker is published for linux amd64/arm64 only, and only by
|
||||
# releases new enough to carry it; skip either case rather than fail
|
||||
# an install that has already put two binaries in place.
|
||||
@@ -388,7 +425,7 @@ main() {
|
||||
if [ "$COMPONENT" = "weed" ] || [ "$COMPONENT" = "all" ]; then
|
||||
info " weed: ${INSTALL_DIR}/weed"
|
||||
fi
|
||||
if [ "$COMPONENT" = "volume-rust" ] || [ "$COMPONENT" = "all" ]; then
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
info " weed-volume: ${INSTALL_DIR}/weed-volume"
|
||||
fi
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
@@ -401,7 +438,9 @@ main() {
|
||||
info "Quick start:"
|
||||
info " weed master # Start master server"
|
||||
info " weed volume -mserver=localhost:9333 # Start Go volume server"
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
if [ "$VOLUME_INSTALLED" = true ]; then
|
||||
info " weed-volume -mserver localhost:9333 # Start Rust volume server"
|
||||
fi
|
||||
if [ "$WORKER_INSTALLED" = true ]; then
|
||||
info " weed-worker --admin localhost:12535 # Start the Rust maintenance worker"
|
||||
fi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.53"
|
||||
appVersion: "4.54"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.53.1
|
||||
version: 4.54.0
|
||||
@@ -45,6 +45,8 @@ Rule of thumb: start with 3 masters, then grow volume servers with data.
|
||||
- Metadata store options: PostgreSQL, MySQL, Redis, Cassandra, TiKV, etc.
|
||||
- Run **3 filers** for HA
|
||||
- Filers are stateless relative to each other — state lives in the DB
|
||||
- No quorum among filers: 2 already survive one failure; the third keeps a spare while one is down for upgrades
|
||||
- Give clients every filer address (e.g. `-filer=f1:8888,f2:8888,f3:8888`) or a load balancer that forwards both the HTTP and gRPC ports (8888 and 18888)
|
||||
|
||||
**S3 Servers**
|
||||
|
||||
@@ -77,7 +79,7 @@ Keeps housekeeping off the hot path.
|
||||
|----------------|----------------------|-----------------------------------|
|
||||
| Master | 3 | Raft quorum, topology, volume map |
|
||||
| Volume Server | N (scale out) | Stores data needles |
|
||||
| Filer | 3 | Metadata gateway |
|
||||
| Filer | 3 (2 minimum) | Metadata gateway, no quorum |
|
||||
| Metadata DB | HA cluster | Persists filer metadata |
|
||||
| S3 Server | N (by throughput) | S3 API frontend |
|
||||
| Admin | 1 | Background job coordinator |
|
||||
|
||||
+1055
-1084
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 55 KiB After Width: | Height: | Size: 54 KiB |
@@ -57,3 +57,61 @@ class VectorInstallerTest(unittest.TestCase):
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix(), SYSTEM=system)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / name).read_bytes(), contents)
|
||||
|
||||
|
||||
class EnterpriseInstallerTest(unittest.TestCase):
|
||||
shell = VectorInstallerTest.shell
|
||||
|
||||
def enterprise(self, code, **env):
|
||||
return self.shell('ENTERPRISE=true; REPO=seaweedfs/artifactory; EDITION=-enterprise; ' + code, **env)
|
||||
|
||||
def test_asset_names(self):
|
||||
result = self.enterprise('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'rust_asset_name linux arm64; worker_asset_name linux amd64; '
|
||||
'vector_asset_name linux arm64; LARGE_DISK=true; '
|
||||
'go_asset_name darwin arm64; rust_asset_name linux amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'weed-enterprise-linux_amd64.tar.gz', 'weed-enterprise-windows_amd64.zip',
|
||||
'weed-volume-enterprise_linux_arm64.tar.gz', 'weed-worker-enterprise_linux_amd64.tar.gz',
|
||||
'weed-vector-enterprise_linux_arm64.tar.gz', 'weed-enterprise-darwin_arm64_large_disk.tar.gz',
|
||||
'weed-volume-enterprise_large_disk_linux_amd64.tar.gz',
|
||||
])
|
||||
|
||||
def test_open_source_go_asset_names(self):
|
||||
result = self.shell('go_asset_name linux amd64; go_asset_name windows amd64; '
|
||||
'LARGE_DISK=true; go_asset_name windows amd64')
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.split(), [
|
||||
'linux_amd64.tar.gz', 'windows_amd64.zip', 'windows_amd64_large_disk.zip',
|
||||
])
|
||||
|
||||
def test_windows_weed_keeps_exe(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
directory = Path(directory)
|
||||
archive = directory / 'weed.zip'
|
||||
with zipfile.ZipFile(archive, 'w') as output:
|
||||
output.writestr('weed.exe', b'weed fixture')
|
||||
dest = directory / 'bin'
|
||||
result = self.shell('download() { cp "$ARCHIVE" "$2"; }; VERSION=test; '
|
||||
'INSTALL_DIR="$DEST"; install_component weed windows amd64',
|
||||
ARCHIVE=archive.as_posix(), DEST=dest.as_posix())
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual((dest / 'weed.exe').read_bytes(), b'weed fixture')
|
||||
|
||||
def test_rust_binaries_linux_only(self):
|
||||
for system, arch, supported in [
|
||||
('linux', 'amd64', True), ('linux', 'arm64', True),
|
||||
('darwin', 'arm64', False), ('windows', 'amd64', False),
|
||||
]:
|
||||
with self.subTest(system=system, arch=arch):
|
||||
for check in ['vector_platform_supported', 'rust_volume_platform_supported']:
|
||||
result = self.enterprise(f'{check} "$SYSTEM" "$ARCH"', SYSTEM=system, ARCH=arch)
|
||||
self.assertEqual(result.returncode == 0, supported, check)
|
||||
|
||||
def test_latest_version_skips_other_releases(self):
|
||||
tags = ['vfs-0.2.5', 'kvcache-filer-native-v0.1.0-preview.1', '4.54-rc1', '4.53', '4.52']
|
||||
listing = ''.join(f' "tag_name": "{tag}",\\n' for tag in tags)
|
||||
result = self.enterprise(f"curl() {{ printf '{listing}'; }}; get_latest_version")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout.strip(), '4.53')
|
||||
@@ -256,6 +256,17 @@ var pluginConfigSections = []pluginConfigSection{
|
||||
"collection_filter": stringValue,
|
||||
},
|
||||
},
|
||||
{
|
||||
// s3_lifecycle has no worker config form; its knobs are admin values
|
||||
jobType: "s3_lifecycle",
|
||||
prefix: "maintenance.s3_lifecycle",
|
||||
adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": int64Value,
|
||||
"walker_interval_minutes": int64Value,
|
||||
"cluster_deletes_per_second": int64Value,
|
||||
"cluster_deletes_burst": int64Value,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func doubleValue(v TomlConfig, key string) *plugin_pb.ConfigValue {
|
||||
|
||||
@@ -401,3 +401,56 @@ max_shard_transfers = 0
|
||||
t.Fatalf("max_shard_transfers = %d, want 0", got.GetInt64Value())
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyPluginConfigFromTomlS3Lifecycle(t *testing.T) {
|
||||
s := newPluginServer(t)
|
||||
seed := &plugin_pb.PersistedJobTypeConfig{
|
||||
JobType: "s3_lifecycle",
|
||||
AdminRuntime: &plugin_pb.AdminRuntimeConfig{Enabled: true, RetryLimit: 1},
|
||||
AdminConfigValues: map[string]*plugin_pb.ConfigValue{
|
||||
"meta_log_retention_days": {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 7}},
|
||||
},
|
||||
}
|
||||
if err := s.GetPlugin().SaveJobTypeConfig(seed); err != nil {
|
||||
t.Fatalf("seed: %v", err)
|
||||
}
|
||||
|
||||
v := tomlConfig(t, `
|
||||
[maintenance.s3_lifecycle]
|
||||
meta_log_retention_days = 0
|
||||
walker_interval_minutes = 30
|
||||
cluster_deletes_per_second = 50
|
||||
cluster_deletes_burst = 200
|
||||
retry_limit = 3
|
||||
`)
|
||||
if err := s.ApplyPluginConfigFromToml(v); err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := s.GetPlugin().LoadJobTypeConfig("s3_lifecycle")
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("load: %v %v", cfg, err)
|
||||
}
|
||||
// an explicit 0 must replace the saved value, not read as unset
|
||||
if got, ok := cfg.AdminConfigValues["meta_log_retention_days"].GetKind().(*plugin_pb.ConfigValue_Int64Value); !ok || got.Int64Value != 0 {
|
||||
t.Errorf("meta_log_retention_days = %v, want int64 0", cfg.AdminConfigValues["meta_log_retention_days"])
|
||||
}
|
||||
if got := cfg.AdminConfigValues["walker_interval_minutes"].GetInt64Value(); got != 30 {
|
||||
t.Errorf("walker_interval_minutes = %d, want 30", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_per_second"].GetInt64Value(); got != 50 {
|
||||
t.Errorf("cluster_deletes_per_second = %d, want 50", got)
|
||||
}
|
||||
if got := cfg.AdminConfigValues["cluster_deletes_burst"].GetInt64Value(); got != 200 {
|
||||
t.Errorf("cluster_deletes_burst = %d, want 200", got)
|
||||
}
|
||||
if cfg.AdminRuntime.RetryLimit != 3 {
|
||||
t.Errorf("retry_limit = %d, want 3", cfg.AdminRuntime.RetryLimit)
|
||||
}
|
||||
if !cfg.AdminRuntime.Enabled {
|
||||
t.Errorf("enabled flipped off by unrelated toml keys")
|
||||
}
|
||||
if cfg.UpdatedBy != "admin.toml" {
|
||||
t.Errorf("updated_by = %q, want admin.toml", cfg.UpdatedBy)
|
||||
}
|
||||
}
|
||||
@@ -76,3 +76,26 @@
|
||||
# retry_limit = 1
|
||||
# seconds to wait between retry attempts
|
||||
# retry_backoff_seconds = 30
|
||||
|
||||
[maintenance.s3_lifecycle]
|
||||
# enabled = true
|
||||
# How far back the filer meta-log subscription can reliably reach, in days.
|
||||
# Rules whose TTL exceeds this window cannot be serviced by replay alone and
|
||||
# are partitioned into the walk set, which steady-state walks scan in full,
|
||||
# subject to walker_interval_minutes.
|
||||
# 0 means "no retention info supplied": the run then falls back to the longest
|
||||
# rule TTL, which keeps every replay rule in replay and promotes nothing.
|
||||
# Set this to the window the filer meta-log actually retains.
|
||||
# meta_log_retention_days = 0
|
||||
# Minimum minutes between steady-state bucket walks per shard. Cold-start and
|
||||
# recovery walks ignore it. 0 walks on every run.
|
||||
# walker_interval_minutes = 0
|
||||
# Cluster-wide delete budget in delete RPCs per second, divided across
|
||||
# execute-capable workers at dispatch time. 0 means unlimited.
|
||||
# cluster_deletes_per_second = 0
|
||||
# Token-bucket burst for the delete budget. 0 means 2 x per-second.
|
||||
# cluster_deletes_burst = 0
|
||||
# max retry attempts for a failed lifecycle run
|
||||
# retry_limit = 0
|
||||
# seconds to wait between retry attempts
|
||||
# retry_backoff_seconds = 5
|
||||
@@ -78,7 +78,10 @@ expires_after_seconds = 10 # seconds
|
||||
# gRPC mTLS configuration
|
||||
# All gRPC TLS authentications are mutual (mTLS)
|
||||
# The values for ca, cert, and key are paths to the certificate/key files
|
||||
# The host name is not checked, so the certificate files can be shared
|
||||
# The host name is not checked, so the certificate files can be shared.
|
||||
# Set verify_server_host = true under [grpc] or a [grpc.<component>] section to
|
||||
# require that a server certificate is valid for the host or IP that component
|
||||
# dials, as the Rust volume server always does.
|
||||
# Each [grpc.<component>] section also accepts optional client_cert/client_key,
|
||||
# presented when that component dials other servers. Set them when your CA
|
||||
# issues separate serverAuth-only and clientAuth-only certificates; when unset,
|
||||
@@ -89,6 +92,7 @@ expires_after_seconds = 10 # seconds
|
||||
# fatal: the component never falls back to plaintext.
|
||||
[grpc]
|
||||
ca = ""
|
||||
verify_server_host = false
|
||||
# Set wildcard domain for enable TLS authentication by common names
|
||||
allowed_wildcard_domain = "" # .mycompany.com
|
||||
# Optional allowlist a client checks the dialed server's certificate against:
|
||||
@@ -97,9 +101,10 @@ allowed_wildcard_domain = "" # .mycompany.com
|
||||
# With a per-workload CA, list the identities each server may present so one
|
||||
# workload cannot impersonate another component. Unset accepts any CA-signed
|
||||
# server certificate, as before. Any [grpc.<component>] section may set its own
|
||||
# allowed_server_commonNames to override this list, and scope it further per
|
||||
# dialed target, e.g. the filer reads [grpc.filer]
|
||||
# volume_allowed_server_commonNames for volume server calls.
|
||||
# allowed_server_commonNames to override this list. Only the filer scopes it
|
||||
# further per dialed target: [grpc.filer] master_, volume_ and
|
||||
# s3_allowed_server_commonNames. Other components dial every server with one
|
||||
# allowlist and refuse to start when per-target keys are their only allowlist.
|
||||
allowed_server_commonNames = "" # comma-separated server identities
|
||||
allowed_server_wildcard_domain = "" # .mycompany.com
|
||||
|
||||
|
||||
@@ -288,6 +288,14 @@ func (f *Filer) newMetadataEvent(oldEntry, newEntry *Entry, deleteChunks, isFrom
|
||||
}
|
||||
}
|
||||
|
||||
// notifyRenameTarget wakes the subscribers of a move's destination directory,
|
||||
// which the partition key (the source directory) does not cover.
|
||||
func notifyRenameTarget(logBuffer *log_buffer.LogBuffer, event *filer_pb.SubscribeMetadataResponse) {
|
||||
if target := filer_pb.MetadataEventTargetDirectory(event); target != event.Directory {
|
||||
logBuffer.NotifySubscribers([]byte(target))
|
||||
}
|
||||
}
|
||||
|
||||
func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetadataResponse) {
|
||||
data, err := proto.Marshal(event)
|
||||
if err != nil {
|
||||
@@ -300,6 +308,7 @@ func (f *Filer) logMetaEvent(ctx context.Context, event *filer_pb.SubscribeMetad
|
||||
glog.Errorf("failed to add data to log buffer for %s: %v", event.Directory, err)
|
||||
return
|
||||
}
|
||||
notifyRenameTarget(f.LocalMetaLogBuffer, event)
|
||||
// The in-memory copy now carries the position the stream assigns it, the
|
||||
// same value subscribers will see delivered.
|
||||
event.TsNs = tsNs
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package filer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||
)
|
||||
|
||||
func TestNotifyRenameTargetWakesDestinationSubscribers(t *testing.T) {
|
||||
logBuffer := log_buffer.NewLogBuffer("test", time.Minute, func(*log_buffer.LogBuffer, time.Time, time.Time, []byte, int64, int64) {}, nil, nil)
|
||||
defer logBuffer.ShutdownLogBuffer()
|
||||
|
||||
logBuffer.SetSubscriberWakeFilter("dst", func(key []byte) bool { return bytes.Equal(key, []byte("/dst")) })
|
||||
dst := logBuffer.RegisterSubscriber("dst")
|
||||
defer logBuffer.UnregisterSubscriber("dst")
|
||||
|
||||
woken := func() bool {
|
||||
select {
|
||||
case <-dst:
|
||||
return true
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
|
||||
EventNotification: &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/src"}})
|
||||
if woken() {
|
||||
t.Fatal("an event that stays in its directory must not wake other directories")
|
||||
}
|
||||
|
||||
notifyRenameTarget(logBuffer, &filer_pb.SubscribeMetadataResponse{Directory: "/src",
|
||||
EventNotification: &filer_pb.EventNotification{OldEntry: &filer_pb.Entry{Name: "a"}, NewEntry: &filer_pb.Entry{Name: "a"}, NewParentPath: "/dst"}})
|
||||
if !woken() {
|
||||
t.Fatal("a move must wake the destination directory's subscribers")
|
||||
}
|
||||
}
|
||||
@@ -628,6 +628,7 @@ func (ma *MetaAggregator) doSubscribeToOneFiler(f *Filer, self pb.ServerAddress,
|
||||
glog.Errorf("failed to add data to log buffer for %s: %v", dir, err)
|
||||
return err
|
||||
}
|
||||
notifyRenameTarget(ma.MetaLogBuffer, event)
|
||||
// The event now sits at tsNs in the local stream; offset tracking
|
||||
// still needs the peer's own position, so it is passed separately.
|
||||
event.TsNs = tsNs
|
||||
|
||||
@@ -54,19 +54,17 @@ func newMemoryChunkPages(fh *FileHandle, chunkSize int64) *ChunkedDirtyPages {
|
||||
return dirtyPages
|
||||
}
|
||||
|
||||
func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
func (pages *ChunkedDirtyPages) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error) {
|
||||
// A failed chunk upload already dropped its data, so this handle can never
|
||||
// flush cleanly. Reject further writes so the writer fails fast instead of
|
||||
// pushing the rest of the file through a pipeline that cannot persist it.
|
||||
if err := pages.LastError(); err != nil {
|
||||
return err
|
||||
return 0, err
|
||||
}
|
||||
pages.hasWrites = true
|
||||
|
||||
glog.V(4).Infof("%v memory AddPage [%d, %d)", pages.fh.fh, offset, offset+int64(len(data)))
|
||||
_, err := pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
|
||||
|
||||
return err
|
||||
return pages.uploadPipeline.SaveDataAt(data, offset, isSequential, tsNs)
|
||||
}
|
||||
|
||||
func (pages *ChunkedDirtyPages) HasWrites() bool {
|
||||
|
||||
@@ -32,7 +32,7 @@ func TestChunkedDirtyPagesFailWritesAfterUploadError(t *testing.T) {
|
||||
uploadErr := errors.New("assign volume failure: no writable volumes")
|
||||
pages.setLastError(uploadErr)
|
||||
|
||||
if err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
|
||||
if _, err := pages.AddPage(0, []byte("x"), true, 1); !errors.Is(err, uploadErr) {
|
||||
t.Fatalf("AddPage after upload failure = %v, want sticky %v", err, uploadErr)
|
||||
}
|
||||
if err := pages.FlushData(); !errors.Is(err, uploadErr) {
|
||||
|
||||
@@ -15,10 +15,6 @@ type PageWriter struct {
|
||||
randomWriter page_writer.DirtyPages
|
||||
}
|
||||
|
||||
var (
|
||||
_ = page_writer.DirtyPages(&PageWriter{})
|
||||
)
|
||||
|
||||
func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
|
||||
pw := &PageWriter{
|
||||
fh: fh,
|
||||
@@ -29,23 +25,27 @@ func newPageWriter(fh *FileHandle, chunkSize int64) *PageWriter {
|
||||
return pw
|
||||
}
|
||||
|
||||
func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
// AddPage returns how many leading bytes of data it buffered; on error those
|
||||
// bytes stay buffered.
|
||||
func (pw *PageWriter) AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (buffered int64, err error) {
|
||||
|
||||
glog.V(4).Infof("%v AddPage [%d, %d)", pw.fh.fh, offset, offset+int64(len(data)))
|
||||
|
||||
chunkIndex := offset / pw.chunkSize
|
||||
for i := chunkIndex; len(data) > 0; i++ {
|
||||
writeSize := min(int64(len(data)), (i+1)*pw.chunkSize-offset)
|
||||
if err := pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs); err != nil {
|
||||
return err
|
||||
n, err := pw.addToOneChunk(i, offset, data[:writeSize], isSequential, tsNs)
|
||||
buffered += int64(n)
|
||||
if err != nil {
|
||||
return buffered, err
|
||||
}
|
||||
offset += writeSize
|
||||
data = data[writeSize:]
|
||||
}
|
||||
return nil
|
||||
return buffered, nil
|
||||
}
|
||||
|
||||
func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) error {
|
||||
func (pw *PageWriter) addToOneChunk(chunkIndex, offset int64, data []byte, isSequential bool, tsNs int64) (int, error) {
|
||||
return pw.randomWriter.AddPage(offset, data, isSequential, tsNs)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
package page_writer
|
||||
|
||||
type DirtyPages interface {
|
||||
AddPage(offset int64, data []byte, isSequential bool, tsNs int64) error
|
||||
AddPage(offset int64, data []byte, isSequential bool, tsNs int64) (n int, err error)
|
||||
FlushData() error
|
||||
HasWrites() bool
|
||||
ReadDirtyDataAt(data []byte, startOffset int64, tsNs int64) (maxStop int64)
|
||||
|
||||
@@ -2,6 +2,7 @@ package page_writer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -144,12 +145,15 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsN
|
||||
pageChunk = NewMemChunk(logicChunkIndex, up.ChunkSize)
|
||||
// fmt.Printf(" create mem chunk %d\n", logicChunkIndex)
|
||||
} else {
|
||||
pageChunk = up.swapFile.NewSwapFileChunk(logicChunkIndex)
|
||||
// Check the *SwapFileChunk before it becomes a PageChunk: a nil
|
||||
// pointer inside the interface would not compare equal to nil.
|
||||
swapChunk := up.swapFile.NewSwapFileChunk(logicChunkIndex)
|
||||
// fmt.Printf(" create file chunk %d\n", logicChunkIndex)
|
||||
if pageChunk == nil {
|
||||
if swapChunk == nil {
|
||||
up.accountant.Release(up.ChunkSize)
|
||||
return 0, fmt.Errorf("failed to create swap file chunk")
|
||||
}
|
||||
pageChunk = swapChunk
|
||||
}
|
||||
up.writableChunks[logicChunkIndex] = pageChunk
|
||||
}
|
||||
@@ -161,6 +165,9 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool, tsN
|
||||
//}
|
||||
n = pageChunk.WriteDataAt(p, off, tsNs)
|
||||
up.maybeMoveToSealed(pageChunk, logicChunkIndex)
|
||||
if n < len(p) {
|
||||
err = fmt.Errorf("chunk %d stored %d of %d bytes: %w", logicChunkIndex, n, len(p), io.ErrShortWrite)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package page_writer
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// When the swap file cannot be created, SaveDataAt must fail the write
|
||||
// instead of keeping a nil swap chunk and writing through it, and give back
|
||||
// the write budget it reserved for the chunk.
|
||||
func TestSaveDataAtFailsWhenSwapFileCannotBeCreated(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
accountant *WriteBufferAccountant
|
||||
}{
|
||||
{name: "no write budget"},
|
||||
{name: "write budget", accountant: NewWriteBufferAccountant(1 << 20)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
up := NewUploadPipeline(nil, 1024, nil, 16, filepath.Join(notADir, "swap"), tc.accountant)
|
||||
|
||||
// A write that is not sequential goes to the swap file.
|
||||
n, err := up.SaveDataAt([]byte("data"), 0, false, 0)
|
||||
if err == nil {
|
||||
t.Fatalf("SaveDataAt = %d, nil; want an error", n)
|
||||
}
|
||||
if _, found := up.writableChunks[0]; found {
|
||||
t.Error("the failed write left a writable chunk behind")
|
||||
}
|
||||
if used := tc.accountant.Used(); used != 0 {
|
||||
t.Errorf("write budget still holds %d bytes", used)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A swap chunk whose file write fails stores nothing, so SaveDataAt must
|
||||
// report the short write instead of claiming the bytes.
|
||||
func TestSaveDataAtReportsFailedSwapWrite(t *testing.T) {
|
||||
up := NewUploadPipeline(nil, 1024, nil, 16, t.TempDir(), nil)
|
||||
defer up.Shutdown()
|
||||
if n, err := up.SaveDataAt([]byte("data"), 0, false, 0); err != nil || n != 4 {
|
||||
t.Fatalf("first write = %d, %v", n, err)
|
||||
}
|
||||
if err := up.swapFile.file.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n, err := up.SaveDataAt([]byte("more"), 4, false, 0); err == nil || n != 0 {
|
||||
t.Fatalf("write through a closed swap file = %d, %v; want 0 and an error", n, err)
|
||||
}
|
||||
}
|
||||
@@ -180,19 +180,23 @@ func (wfs *WFS) CopyFileRange(cancel <-chan struct{}, in *fuse.CopyFileRangeIn)
|
||||
|
||||
// Perform the write
|
||||
fhOut.dirtyPages.writerPattern.MonitorWriteAt(offsetOut, int(numBytesRead))
|
||||
if err := fhOut.dirtyPages.AddPage(
|
||||
buffered, err := fhOut.dirtyPages.AddPage(
|
||||
offsetOut,
|
||||
buff[:numBytesRead],
|
||||
fhOut.dirtyPages.writerPattern.IsSequentialMode(),
|
||||
nowUnixNano); err != nil {
|
||||
nowUnixNano)
|
||||
if buffered > 0 {
|
||||
fhOut.pageWrites.Add(1)
|
||||
totalCopied += buffered
|
||||
fhOut.dirtyContent = true
|
||||
}
|
||||
if err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
if totalCopied == 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
break
|
||||
}
|
||||
fhOut.pageWrites.Add(1)
|
||||
|
||||
// Accumulate for the next loop iteration
|
||||
totalCopied += numBytesRead
|
||||
fhOut.dirtyContent = true
|
||||
}
|
||||
|
||||
if totalCopied == 0 {
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
@@ -408,6 +410,47 @@ func TestStreamingCopyMarksOnlyAcceptedContentDirty(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Bytes the copy buffered before a page failed stay in the handle, so the
|
||||
// copy reports them and grows the destination over them.
|
||||
func TestStreamingCopyReportsPartlyBufferedCopy(t *testing.T) {
|
||||
wfs := newCopyRangeTestWFS()
|
||||
wfs.option.ConcurrentWriters = 1
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
wfs.option.uniqueCacheDirForWrite = filepath.Join(notADir, "swap")
|
||||
srcInode := wfs.inodeToPath.Lookup(util.FullPath("/src.txt"), 1, false, false, 0, true)
|
||||
dstInode := wfs.inodeToPath.Lookup(util.FullPath("/dst.txt"), 1, false, false, 0, true)
|
||||
srcHandle, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, srcInode, &filer_pb.Entry{
|
||||
Name: "src.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, FileSize: 1024, Inode: srcInode},
|
||||
Content: bytes.Repeat([]byte{'s'}, 1024),
|
||||
}, 0, 0)
|
||||
dstHandle, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, dstInode, &filer_pb.Entry{
|
||||
Name: "dst.txt",
|
||||
Attributes: &filer_pb.FuseAttributes{FileMode: 0100644, Inode: dstInode},
|
||||
}, 0, 0)
|
||||
t.Cleanup(dstHandle.dirtyPages.Destroy)
|
||||
|
||||
// The destination range crosses into a second chunk, which needs the swap file.
|
||||
written, status := wfs.CopyFileRange(make(chan struct{}), &fuse.CopyFileRangeIn{
|
||||
FhIn: uint64(srcHandle.fh),
|
||||
FhOut: uint64(dstHandle.fh),
|
||||
OffOut: 512,
|
||||
Len: 1024,
|
||||
})
|
||||
if status != fuse.OK || written != 512 {
|
||||
t.Fatalf("CopyFileRange = %d, %v; want a short copy of 512", written, status)
|
||||
}
|
||||
if size := dstHandle.GetEntry().GetEntry().GetAttributes().GetFileSize(); size != 1024 {
|
||||
t.Fatalf("destination size = %d, want 1024", size)
|
||||
}
|
||||
if !dstHandle.dirtyContent || !dstHandle.dirtyMetadata {
|
||||
t.Fatal("the buffered prefix must leave the destination dirty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestServerSideCopySkipsFencedDestination(t *testing.T) {
|
||||
wfs := newCopyRangeTestWFS()
|
||||
wfs.option.WritebackCache = true
|
||||
|
||||
@@ -270,7 +270,7 @@ func TestTruncateEntryClearsDirtyPagesForOpenHandle(t *testing.T) {
|
||||
fh, _, _, _ := wfs.fhMap.AcquireFileHandle(wfs, inode, entry, 0, 0)
|
||||
fh.RememberPath(fullPath)
|
||||
|
||||
if err := fh.dirtyPages.AddPage(0, []byte("hello"), true, time.Now().UnixNano()); err != nil {
|
||||
if _, err := fh.dirtyPages.AddPage(0, []byte("hello"), true, time.Now().UnixNano()); err != nil {
|
||||
t.Fatalf("AddPage: %v", err)
|
||||
}
|
||||
oldDirtyPages := fh.dirtyPages
|
||||
|
||||
@@ -68,6 +68,21 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
return 0, fuse.OK
|
||||
}
|
||||
|
||||
// Size and times cover only the bytes buffered, or the next flush
|
||||
// publishes a hole where the data should be. A partly buffered write
|
||||
// is reported as a short write, or as an error to kernel writeback,
|
||||
// which ignores the count.
|
||||
offset := int64(in.Offset)
|
||||
buffered, err := fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode(), tsNs)
|
||||
if err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
if buffered == 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
data = data[:buffered]
|
||||
}
|
||||
fh.pageWrites.Add(1)
|
||||
|
||||
// Only an append past all locally owned bytes needs the stored EOF guard.
|
||||
if in.Offset > entry.Attributes.FileSize && wfs.isAppendWrite(in) {
|
||||
if !fh.pendingAppend || in.Offset < fh.pendingAppendOffset {
|
||||
@@ -76,7 +91,6 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
fh.pendingAppend = true
|
||||
}
|
||||
entry.Content = nil
|
||||
offset := int64(in.Offset)
|
||||
oldFileSize := int64(entry.Attributes.FileSize)
|
||||
newFileSize := max(offset+int64(len(data)), oldFileSize)
|
||||
entry.Attributes.FileSize = uint64(newFileSize)
|
||||
@@ -96,14 +110,6 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
wfs.AddUncommittedBytes(newFileSize - oldFileSize)
|
||||
}
|
||||
|
||||
// glog.V(4).Infof("%v write [%d,%d) %d", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)), len(req.Data))
|
||||
|
||||
if err := fh.dirtyPages.AddPage(offset, data, fh.dirtyPages.writerPattern.IsSequentialMode(), tsNs); err != nil {
|
||||
glog.Errorf("AddPage error: %v", err)
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
fh.pageWrites.Add(1)
|
||||
|
||||
written = uint32(len(data))
|
||||
|
||||
if offset == 0 {
|
||||
@@ -127,5 +133,8 @@ func (wfs *WFS) Write(cancel <-chan struct{}, in *fuse.WriteIn, data []byte) (wr
|
||||
fh.mirrorFile.WriteAt(data, offset)
|
||||
}
|
||||
|
||||
if err != nil && in.WriteFlags&fuse.WRITE_CACHE != 0 {
|
||||
return 0, writeErrorToFuseStatus(err)
|
||||
}
|
||||
return written, fuse.OK
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
//go:build !freebsd
|
||||
|
||||
package mount
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// A write the swap file cannot buffer fails, and must not grow the file: the
|
||||
// next flush would publish that size with a hole where its data should be. A
|
||||
// write that buffers only its first chunk is a short write of those bytes, or
|
||||
// an error for kernel writeback, which would drop the rest silently.
|
||||
func TestFailedWriteDoesNotGrowFile(t *testing.T) {
|
||||
t.Run("short write", func(t *testing.T) { testFailedWriteDoesNotGrowFile(t, 0) })
|
||||
t.Run("kernel writeback", func(t *testing.T) { testFailedWriteDoesNotGrowFile(t, fuse.WRITE_CACHE) })
|
||||
}
|
||||
|
||||
func testFailedWriteDoesNotGrowFile(t *testing.T, writeFlags uint32) {
|
||||
base := attributeAppendEntry(1)
|
||||
base.CommitTsNs = 100
|
||||
peer := &g6ConditionalPeer{entry: proto.Clone(base).(*filer_pb.Entry)}
|
||||
peer.blobs = map[string][]byte{base.Chunks[0].GetFileIdString(): bytes.Repeat([]byte{'O'}, 256)}
|
||||
|
||||
w := newInvalidateTestWFS(t)
|
||||
notADir := filepath.Join(t.TempDir(), "file")
|
||||
if err := os.WriteFile(notADir, nil, 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.option.uniqueCacheDirForWrite = filepath.Join(notADir, "swap")
|
||||
w.option.DefaultPermissions = true
|
||||
w.posixLocks = NewPosixLockTable()
|
||||
w.option.ConcurrentWriters = 1
|
||||
w.concurrentWriters = util.NewLimitedConcurrentExecutor(1)
|
||||
startFakeFiler(t, w, peer)
|
||||
w.inodeToPath.Lookup("/dir", 1, true, false, 0, false)
|
||||
w.inodeToPath.Lookup("/dir/file", 1, false, false, 42, true)
|
||||
w.inodeToPath.MarkChildrenCached("/dir")
|
||||
if err := w.metaCache.InsertEntry(context.Background(), filer.FromPbEntry("/dir", proto.Clone(base).(*filer_pb.Entry)), 1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := &fuse.OpenOut{}
|
||||
if code := w.Open(nil, &fuse.OpenIn{InHeader: fuse.InHeader{NodeId: 42}, Flags: syscall.O_WRONLY}, out); code != fuse.OK {
|
||||
t.Fatal(code)
|
||||
}
|
||||
fh := w.GetHandle(FileHandleId(out.Fh))
|
||||
g6AppendUploader(t, fh, 100, peer)
|
||||
|
||||
write := func(offset uint64, size int, value byte) (uint32, fuse.Status) {
|
||||
return w.Write(nil, &fuse.WriteIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: out.Fh, Offset: offset, Size: uint32(size), WriteFlags: writeFlags}, bytes.Repeat([]byte{value}, size))
|
||||
}
|
||||
if _, code := write(256, 256, 'A'); code != fuse.OK {
|
||||
t.Fatalf("buffered write: %v", code)
|
||||
}
|
||||
// The only writable chunk slot is taken, so this write needs the swap file.
|
||||
if _, code := write(4096, 256, 'B'); code == fuse.OK {
|
||||
t.Fatal("write through an unusable swap directory succeeded")
|
||||
}
|
||||
if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 512 {
|
||||
t.Fatalf("file size after the failed write = %d, want 512", size)
|
||||
}
|
||||
// The first 24 bytes fit the existing chunk; the next chunk needs the swap file.
|
||||
n, code := write(1000, 48, 'C')
|
||||
if writeFlags == 0 && (code != fuse.OK || n != 24) {
|
||||
t.Fatalf("boundary-crossing write = %d, %v; want a short write of 24", n, code)
|
||||
}
|
||||
if writeFlags != 0 && code == fuse.OK {
|
||||
t.Fatalf("boundary-crossing writeback = %d, OK; want an error", n)
|
||||
}
|
||||
if size := fh.GetEntry().GetEntry().Attributes.FileSize; size != 1024 {
|
||||
t.Fatalf("file size after the short write = %d, want 1024", size)
|
||||
}
|
||||
if code := w.Fsync(nil, &fuse.FsyncIn{InHeader: fuse.InHeader{NodeId: 42}, Fh: out.Fh}); code != fuse.OK {
|
||||
t.Fatal(code)
|
||||
}
|
||||
|
||||
peer.mu.Lock()
|
||||
defer peer.mu.Unlock()
|
||||
want := append(bytes.Repeat([]byte{'O'}, 256), bytes.Repeat([]byte{'A'}, 256)...)
|
||||
want = append(want, make([]byte, 1000-512)...)
|
||||
want = append(want, bytes.Repeat([]byte{'C'}, 24)...)
|
||||
if got := g6StoredBytes(t, peer); peer.entry.Attributes.FileSize != 1024 || !bytes.Equal(got, want) {
|
||||
t.Fatalf("stored size %d, want 1024 with the original and buffered bytes", peer.entry.Attributes.FileSize)
|
||||
}
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func newSlowDataFlush(t *testing.T, assignDelay, deadline time.Duration, stuckCo
|
||||
Chunks: stored,
|
||||
}, 0, 0)
|
||||
t.Cleanup(func() { wfs.ReleaseHandle(fh.fh) })
|
||||
if err := fh.dirtyPages.AddPage(offset, data, true, time.Now().UnixNano()); err != nil {
|
||||
if _, err := fh.dirtyPages.AddPage(offset, data, true, time.Now().UnixNano()); err != nil {
|
||||
t.Fatalf("AddPage: %v", err)
|
||||
}
|
||||
fh.dirtyMetadata = true
|
||||
|
||||
@@ -57,6 +57,54 @@ func ScanMetadataEventSkeleton(data []byte) (skeleton *SubscribeMetadataResponse
|
||||
return skeleton, true
|
||||
}
|
||||
|
||||
// ScanMetadataEventDirectories extracts an event's directory and its
|
||||
// new_parent_path without allocating. ok=false means the data is malformed.
|
||||
func ScanMetadataEventDirectories(data []byte) (directory, newParentPath []byte, ok bool) {
|
||||
rest := data
|
||||
for len(rest) > 0 {
|
||||
num, typ, n := protowire.ConsumeTag(rest)
|
||||
if n < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[n:]
|
||||
if typ != protowire.BytesType || (num != 1 && num != 2) {
|
||||
m := protowire.ConsumeFieldValue(num, typ, rest)
|
||||
if m < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[m:]
|
||||
continue
|
||||
}
|
||||
v, m := protowire.ConsumeBytes(rest)
|
||||
if m < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
rest = rest[m:]
|
||||
if num == 1 { // directory
|
||||
directory = v
|
||||
continue
|
||||
}
|
||||
// event_notification: repeated occurrences merge, so the last
|
||||
// new_parent_path wins
|
||||
for len(v) > 0 {
|
||||
fnum, ftyp, fn := protowire.ConsumeTag(v)
|
||||
if fn < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
v = v[fn:]
|
||||
fm := protowire.ConsumeFieldValue(fnum, ftyp, v)
|
||||
if fm < 0 {
|
||||
return nil, nil, false
|
||||
}
|
||||
if fnum == 4 && ftyp == protowire.BytesType { // new_parent_path
|
||||
newParentPath, _ = protowire.ConsumeBytes(v)
|
||||
}
|
||||
v = v[fm:]
|
||||
}
|
||||
}
|
||||
return directory, newParentPath, true
|
||||
}
|
||||
|
||||
func scanEventNotificationSkeleton(data []byte) (*EventNotification, bool) {
|
||||
notification := &EventNotification{}
|
||||
rest := data
|
||||
|
||||
@@ -110,3 +110,86 @@ func TestScanMetadataEventSkeletonEmptyPayload(t *testing.T) {
|
||||
t.Fatalf("empty payload: %+v ok=%v", skeleton, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetadataDirectoryMayMatchSubscriptionCoversMatches(t *testing.T) {
|
||||
dirs := []string{"/", "/data", "/data/pvc-7", "/data/pvc-70", "/data/pvc-7/sub", "/data/pvc-1", "/other", "/data/./pvc-7", "/data//pvc-7", "/data/pvc-1/../pvc-7"}
|
||||
names := []string{"pvc-7", "pvc-70", "a.log", "sub", ""}
|
||||
filters := []struct {
|
||||
prefix string
|
||||
prefixes []string
|
||||
dirs []string
|
||||
}{
|
||||
{prefix: "/data/pvc-7/"},
|
||||
{prefix: "/data/pvc-7"},
|
||||
{prefix: "/data/"},
|
||||
{prefix: "/"},
|
||||
{prefix: "/data/pvc-7/sub/deeper/"},
|
||||
{prefix: "/x", prefixes: []string{"/other/", "/data/pvc-7/"}},
|
||||
{prefix: "/x", dirs: []string{"/data/pvc-1"}},
|
||||
}
|
||||
for _, dir := range dirs {
|
||||
for _, target := range append([]string{""}, dirs...) {
|
||||
for _, name := range names {
|
||||
event := &SubscribeMetadataResponse{Directory: dir, EventNotification: &EventNotification{
|
||||
OldEntry: &Entry{Name: name}, NewEntry: &Entry{Name: name}, NewParentPath: target}}
|
||||
for _, f := range filters {
|
||||
if !MetadataEventMatchesSubscription(event, f.prefix, f.prefixes, f.dirs) {
|
||||
continue
|
||||
}
|
||||
if !MetadataDirectoryMayMatchSubscription([]byte(dir), f.prefix, f.prefixes, f.dirs) &&
|
||||
(target == "" || !MetadataDirectoryMayMatchSubscription([]byte(target), f.prefix, f.prefixes, f.dirs)) {
|
||||
t.Errorf("event %q/%q -> %q matches %+v but neither directory may match", dir, name, target, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range []struct {
|
||||
dir, prefix string
|
||||
want bool
|
||||
}{
|
||||
{"/data/pvc-1", "/data/pvc-7/", false},
|
||||
{"/data/pvc-70", "/data/pvc-7/", false},
|
||||
{"/other", "/data/", false},
|
||||
{"/data/pvc-7", "/data/pvc-7/", true},
|
||||
{"/data/pvc-7/sub", "/data/pvc-7/", true},
|
||||
{"/data", "/data/pvc-7", true},
|
||||
{"/", "/data/", true},
|
||||
} {
|
||||
if got := MetadataDirectoryMayMatchSubscription([]byte(c.dir), c.prefix, nil, nil); got != c.want {
|
||||
t.Errorf("dir %q prefix %q: got %v, want %v", c.dir, c.prefix, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanMetadataEventDirectories(t *testing.T) {
|
||||
for _, event := range []*SubscribeMetadataResponse{
|
||||
{Directory: "/data/pvc-7", EventNotification: &EventNotification{NewEntry: bigEntry("a.log")}},
|
||||
{Directory: "/data/pvc-7", EventNotification: &EventNotification{OldEntry: bigEntry("a.log"), NewEntry: bigEntry("a.log"), NewParentPath: "/data/pvc-9"}},
|
||||
{Directory: "/data/pvc-7"},
|
||||
{},
|
||||
} {
|
||||
dir, target, ok := ScanMetadataEventDirectories(marshalEvent(t, event))
|
||||
if !ok || string(dir) != event.Directory || string(target) != event.GetEventNotification().GetNewParentPath() {
|
||||
t.Errorf("scan of %+v: dir %q target %q ok %v", event, dir, target, ok)
|
||||
}
|
||||
}
|
||||
|
||||
// repeated event_notification fields merge: the last new_parent_path wins
|
||||
first := marshalEvent(t, &SubscribeMetadataResponse{Directory: "/a", EventNotification: &EventNotification{NewParentPath: "/b"}})
|
||||
second := marshalEvent(t, &SubscribeMetadataResponse{EventNotification: &EventNotification{NewEntry: &Entry{Name: "x"}, NewParentPath: "/c"}})
|
||||
merged := append(append([]byte{}, first...), second...)
|
||||
full := &SubscribeMetadataResponse{}
|
||||
if err := proto.Unmarshal(merged, full); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
dir, target, ok := ScanMetadataEventDirectories(merged)
|
||||
if !ok || string(dir) != full.Directory || string(target) != full.EventNotification.NewParentPath {
|
||||
t.Errorf("merged: dir %q target %q ok %v, full decode %q %q", dir, target, ok, full.Directory, full.EventNotification.NewParentPath)
|
||||
}
|
||||
|
||||
if _, _, ok := ScanMetadataEventDirectories([]byte{0xff, 0xff, 0xff}); ok {
|
||||
t.Error("malformed payload must not scan")
|
||||
}
|
||||
}
|
||||
@@ -414,6 +414,44 @@ func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes
|
||||
return strings.HasPrefix(fullPath, pathPrefix)
|
||||
}
|
||||
|
||||
// MetadataDirectoryMayMatchSubscription reports whether an event in dir could
|
||||
// match the subscription, before the event is decoded. It errs toward true.
|
||||
func MetadataDirectoryMayMatchSubscription(dir []byte, pathPrefix string, pathPrefixes []string, directories []string) bool {
|
||||
if pathPrefix == "" || len(dir) == 0 || directoryMayHavePrefix(dir, pathPrefix) {
|
||||
return true
|
||||
}
|
||||
for _, p := range pathPrefixes {
|
||||
if directoryMayHavePrefix(dir, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
for _, d := range directories {
|
||||
if string(dir) == d {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// matching cleans the joined path, so only a clean dir can be ruled out
|
||||
for i := 0; i+1 < len(dir); i++ {
|
||||
if dir[i] == '/' && (dir[i+1] == '/' || dir[i+1] == '.') {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// directoryMayHavePrefix reports whether some path dir/<name> can start with
|
||||
// prefix: dir+"/" and prefix agree on their common length.
|
||||
func directoryMayHavePrefix(dir []byte, prefix string) bool {
|
||||
n := min(len(dir), len(prefix))
|
||||
if string(dir[:n]) != prefix[:n] {
|
||||
return false
|
||||
}
|
||||
if len(prefix) <= len(dir) || dir[len(dir)-1] == '/' {
|
||||
return true
|
||||
}
|
||||
return prefix[len(dir)] == '/'
|
||||
}
|
||||
|
||||
func hasPrefixIn(text string, prefixes []string) bool {
|
||||
for _, p := range prefixes {
|
||||
if strings.HasPrefix(text, p) {
|
||||
|
||||
@@ -95,7 +95,7 @@ func (s3a *S3ApiServer) createMultipartUpload(r *http.Request, input *s3.CreateM
|
||||
// Prepare error handling outside callback scope
|
||||
var encryptionError error
|
||||
|
||||
if err := s3a.mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
||||
if err := s3a.uploadFiler(*input.Bucket, *input.Key).mkdir(s3a.genUploadsFolder(*input.Bucket), uploadIdString, func(entry *filer_pb.Entry) {
|
||||
if entry.Extended == nil {
|
||||
entry.Extended = make(map[string][]byte)
|
||||
}
|
||||
@@ -388,14 +388,14 @@ func applyMultipartSSES3HeadersFromUploadEntry(dst *filer_pb.Entry, sses3Info *m
|
||||
|
||||
func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload, uploadDirectory, entryName, dirName string, completedPartNumbers []int, completedPartMap map[int][]string, maxPartNo int) (*multipartCompletionState, *CompleteMultipartUploadResult, s3err.ErrorCode) {
|
||||
if output, _ := s3a.committedCompletion(r, input); output != nil {
|
||||
cleanupEntries, _, cleanupErr := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
cleanupEntries, _, cleanupErr := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
if cleanupErr != nil && !errors.Is(cleanupErr, filer_pb.ErrNotFound) {
|
||||
glog.Warningf("completeMultipartUpload: failed to list stale upload directory %s for cleanup: %v", uploadDirectory, cleanupErr)
|
||||
}
|
||||
return &multipartCompletionState{deleteEntries: cleanupEntries, metadataOnlyCleanup: true}, output, s3err.ErrNone
|
||||
}
|
||||
|
||||
entries, _, err := s3a.list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
entries, _, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(uploadDirectory, "", "", false, s3_constants.MaxS3MultipartParts+1)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
|
||||
if isFilerNotFound(err) {
|
||||
@@ -416,7 +416,7 @@ func (s3a *S3ApiServer) prepareMultipartCompletionState(r *http.Request, input *
|
||||
return nil, nil, s3err.ErrNoSuchUpload
|
||||
}
|
||||
|
||||
pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s %s error: %v", *input.Bucket, *input.UploadId, err)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
@@ -796,7 +796,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// The transaction removes the upload directory metadata-only,
|
||||
// so the part entries the object does not reference are freed
|
||||
// first or their chunks leak.
|
||||
if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil {
|
||||
if err := s3a.deleteUnusedPartEntries(r.Context(), uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -807,7 +807,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// Roll back only while the upload directory survives: once the
|
||||
// transaction removed it, the version file is the only record
|
||||
// left and deleting it would make a retry impossible.
|
||||
if exists, _ := s3a.exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists {
|
||||
if exists, _ := s3a.uploadFiler(*input.Bucket, *input.Key).exists(s3a.genUploadsFolder(*input.Bucket), *input.UploadId, true); exists {
|
||||
if rollbackErr := s3a.rollbackMultipartVersion(versionDir, versionFileName); rollbackErr != nil {
|
||||
glog.Errorf("completeMultipartUpload: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, *input.Bucket, *input.Key, rollbackErr)
|
||||
completionState.manifestsReferenced = true
|
||||
@@ -849,7 +849,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
|
||||
if versioningState == s3_constants.VersioningSuspended {
|
||||
// For suspended versioning, add "null" version ID metadata and return "null" version ID
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState)
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
@@ -945,7 +945,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
}
|
||||
|
||||
// For non-versioned buckets, create main object file
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.UploadId, completionState)
|
||||
removal, err := s3a.routedUploadRemoval(r.Context(), owner, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState)
|
||||
if err != nil {
|
||||
glog.Errorf("completeMultipartUpload %s upload %s unused part cleanup: %v", *input.Bucket, *input.UploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
@@ -1073,9 +1073,9 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
cleanupCtx := withFilerRetryBudget(context.Background(), filerRetryRequestBudget)
|
||||
// Keep the directory when an entry delete failed so its metadata
|
||||
// still references the chunks; removing it metadata-only orphans them.
|
||||
if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.UploadId, completionState); err != nil {
|
||||
if err := s3a.deleteUnusedPartEntries(cleanupCtx, uploadDirectory, *input.Bucket, *input.Key, *input.UploadId, completionState); err != nil {
|
||||
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
|
||||
} else if err := s3a.rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
|
||||
} else if err := s3a.uploadFiler(*input.Bucket, *input.Key).rm(cleanupCtx, s3a.genUploadsFolder(*input.Bucket), *input.UploadId, false, true); err != nil {
|
||||
glog.V(1).Infof("completeMultipartUpload cleanup %s upload %s: %v", *input.Bucket, *input.UploadId, err)
|
||||
}
|
||||
}
|
||||
@@ -1091,10 +1091,10 @@ func (s3a *S3ApiServer) completeMultipartUpload(r *http.Request, input *s3.Compl
|
||||
// reference. A finalize that removes the whole upload directory metadata-only
|
||||
// frees them first, or their chunks leak; a failure here must abort the
|
||||
// completion, since the surviving entries keep the upload retriable.
|
||||
func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, uploadId string, completionState *multipartCompletionState) error {
|
||||
func (s3a *S3ApiServer) deleteUnusedPartEntries(ctx context.Context, uploadDirectory, bucket, object, uploadId string, completionState *multipartCompletionState) error {
|
||||
var lastErr error
|
||||
for _, deleteEntry := range completionState.deleteEntries {
|
||||
if err := s3a.rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(ctx, uploadDirectory, deleteEntry.Name, !completionState.metadataOnlyCleanup, true); err != nil {
|
||||
glog.Warningf("completeMultipartUpload cleanup %s upload %s unused %s : %v", bucket, uploadId, deleteEntry.Name, err)
|
||||
lastErr = err
|
||||
}
|
||||
@@ -1182,7 +1182,7 @@ func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput
|
||||
glog.V(2).Infof("abortMultipartUpload input %v", input)
|
||||
|
||||
uploadsFolder := s3a.genUploadsFolder(*input.Bucket)
|
||||
uploadEntry, err := s3a.getEntry(uploadsFolder, *input.UploadId)
|
||||
uploadEntry, err := s3a.uploadFiler(*input.Bucket, aws.StringValue(input.Key)).getEntry(uploadsFolder, *input.UploadId)
|
||||
// complete and abort both remove .uploads/<id>; a missing record is an upload
|
||||
// that does not exist (NoSuchUpload on AWS), not one this call discarded
|
||||
if err != nil {
|
||||
@@ -1215,7 +1215,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
if completed {
|
||||
if err := s3a.rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(context.Background(), s3a.genUploadsFolder(bucket), uploadId, false, true); err != nil {
|
||||
glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -1233,7 +1233,7 @@ func (s3a *S3ApiServer) removeUploadDir(bucket, uploadId, object string) s3err.E
|
||||
func (s3a *S3ApiServer) routedUploadDirDelete(bucket, uploadId, object string) s3err.ErrorCode {
|
||||
uploadsFolder := s3a.genUploadsFolder(bucket)
|
||||
rmUploadDir := func(isDeleteData bool) s3err.ErrorCode {
|
||||
if err := s3a.rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil {
|
||||
if err := s3a.uploadFiler(bucket, object).rm(context.Background(), uploadsFolder, uploadId, isDeleteData, true); err != nil {
|
||||
glog.V(1).Infof("bucket %s remove upload %s: %v", bucket, uploadId, err)
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
@@ -1588,7 +1588,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
|
||||
// directory as empty instead of erroring, so listing alone cannot tell a
|
||||
// completed upload (NoSuchUpload on AWS) from an open upload with no parts
|
||||
// yet (200 with an empty list). Probe the upload record instead.
|
||||
pentry, err := s3a.getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
pentry, err := s3a.uploadFiler(*input.Bucket, *input.Key).getEntry(s3a.genUploadsFolder(*input.Bucket), *input.UploadId)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, s3err.ErrNoSuchUpload
|
||||
@@ -1618,7 +1618,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
|
||||
output.IsTruncated = aws.Bool(false)
|
||||
return output, s3err.ErrNone
|
||||
}
|
||||
entries, isLast, err := s3a.list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts))
|
||||
entries, isLast, err := s3a.uploadFiler(*input.Bucket, *input.Key).list(s3a.genUploadsFolder(*input.Bucket)+"/"+*input.UploadId, "", fmt.Sprintf("%04d", *input.PartNumberMarker+1), true, uint32(*input.MaxParts))
|
||||
if err != nil {
|
||||
// A store that reports the missing upload directory as not-found means
|
||||
// the upload is gone (completed or aborted), not a store error.
|
||||
|
||||
@@ -3,10 +3,8 @@ package policy_engine
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -14,148 +12,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/wildcard"
|
||||
)
|
||||
|
||||
// LRUNode represents a node in the doubly-linked list for efficient LRU operations
|
||||
type LRUNode struct {
|
||||
key string
|
||||
value []string
|
||||
prev *LRUNode
|
||||
next *LRUNode
|
||||
}
|
||||
|
||||
// NormalizedValueCache provides size-limited caching for normalized values with efficient LRU eviction
|
||||
type NormalizedValueCache struct {
|
||||
mu sync.RWMutex
|
||||
cache map[string]*LRUNode
|
||||
maxSize int
|
||||
head *LRUNode // Most recently used
|
||||
tail *LRUNode // Least recently used
|
||||
}
|
||||
|
||||
// NewNormalizedValueCache creates a new normalized value cache with configurable size
|
||||
func NewNormalizedValueCache(maxSize int) *NormalizedValueCache {
|
||||
if maxSize <= 0 {
|
||||
maxSize = 1000 // Default size
|
||||
}
|
||||
|
||||
// Create dummy head and tail nodes for easier list manipulation
|
||||
head := &LRUNode{}
|
||||
tail := &LRUNode{}
|
||||
head.next = tail
|
||||
tail.prev = head
|
||||
|
||||
return &NormalizedValueCache{
|
||||
cache: make(map[string]*LRUNode),
|
||||
maxSize: maxSize,
|
||||
head: head,
|
||||
tail: tail,
|
||||
}
|
||||
}
|
||||
|
||||
// Get retrieves a cached value and updates access order in O(1) time
|
||||
func (c *NormalizedValueCache) Get(key string) ([]string, bool) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if node, exists := c.cache[key]; exists {
|
||||
// Move to head (most recently used) - O(1) operation
|
||||
c.moveToHead(node)
|
||||
return node.value, true
|
||||
}
|
||||
return nil, false
|
||||
}
|
||||
|
||||
// Set stores a value in the cache with size limit enforcement in O(1) time
|
||||
func (c *NormalizedValueCache) Set(key string, value []string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if node, exists := c.cache[key]; exists {
|
||||
// Update existing node and move to head
|
||||
node.value = value
|
||||
c.moveToHead(node)
|
||||
return
|
||||
}
|
||||
|
||||
// Create new node
|
||||
newNode := &LRUNode{
|
||||
key: key,
|
||||
value: value,
|
||||
}
|
||||
|
||||
// If at max size, evict least recently used
|
||||
if len(c.cache) >= c.maxSize {
|
||||
c.evictLeastRecentlyUsed()
|
||||
}
|
||||
|
||||
// Add to cache and move to head
|
||||
c.cache[key] = newNode
|
||||
c.addToHead(newNode)
|
||||
}
|
||||
|
||||
// moveToHead moves a node to the head of the list (most recently used) - O(1)
|
||||
func (c *NormalizedValueCache) moveToHead(node *LRUNode) {
|
||||
c.removeNode(node)
|
||||
c.addToHead(node)
|
||||
}
|
||||
|
||||
// addToHead adds a node right after the head - O(1)
|
||||
func (c *NormalizedValueCache) addToHead(node *LRUNode) {
|
||||
node.prev = c.head
|
||||
node.next = c.head.next
|
||||
c.head.next.prev = node
|
||||
c.head.next = node
|
||||
}
|
||||
|
||||
// removeNode removes a node from the list - O(1)
|
||||
func (c *NormalizedValueCache) removeNode(node *LRUNode) {
|
||||
node.prev.next = node.next
|
||||
node.next.prev = node.prev
|
||||
}
|
||||
|
||||
// removeTail removes the last node before tail (least recently used) - O(1)
|
||||
func (c *NormalizedValueCache) removeTail() *LRUNode {
|
||||
lastNode := c.tail.prev
|
||||
c.removeNode(lastNode)
|
||||
return lastNode
|
||||
}
|
||||
|
||||
// evictLeastRecentlyUsed removes the least recently used item in O(1) time
|
||||
func (c *NormalizedValueCache) evictLeastRecentlyUsed() {
|
||||
tail := c.removeTail()
|
||||
delete(c.cache, tail.key)
|
||||
}
|
||||
|
||||
// Global cache instance with size limit
|
||||
var normalizedValueCache = NewNormalizedValueCache(1000)
|
||||
|
||||
// getCachedNormalizedValues returns cached normalized values or caches new ones
|
||||
func getCachedNormalizedValues(value interface{}) []string {
|
||||
if value == nil {
|
||||
return nil // no expected values; never reach reflect.TypeOf(nil)
|
||||
}
|
||||
// Create a string key for caching - more efficient than fmt.Sprintf
|
||||
typeStr := reflect.TypeOf(value).String()
|
||||
cacheKey := typeStr + ":" + fmt.Sprint(value)
|
||||
|
||||
// Try to get from cache
|
||||
if cached, exists := normalizedValueCache.Get(cacheKey); exists {
|
||||
return cached
|
||||
}
|
||||
|
||||
// Not in cache, normalize and store
|
||||
// Use the error-handling version for better error reporting
|
||||
normalized, err := normalizeToStringSliceWithError(value)
|
||||
if err != nil {
|
||||
glog.Warningf("Failed to normalize policy value %v: %v", value, err)
|
||||
// Fallback to string conversion for backward compatibility
|
||||
normalized = []string{fmt.Sprintf("%v", value)}
|
||||
}
|
||||
|
||||
normalizedValueCache.Set(cacheKey, normalized)
|
||||
|
||||
return normalized
|
||||
}
|
||||
|
||||
// ConditionEvaluator evaluates policy conditions
|
||||
type ConditionEvaluator interface {
|
||||
Evaluate(conditionValue interface{}, contextValues []string) bool
|
||||
@@ -165,7 +21,7 @@ type ConditionEvaluator interface {
|
||||
type StringEqualsEvaluator struct{}
|
||||
|
||||
func (e *StringEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -180,7 +36,7 @@ func (e *StringEqualsEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type StringNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *StringNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -195,7 +51,7 @@ func (e *StringNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type StringLikeEvaluator struct{}
|
||||
|
||||
func (e *StringLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -210,7 +66,7 @@ func (e *StringLikeEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type StringNotLikeEvaluator struct{}
|
||||
|
||||
func (e *StringNotLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -225,7 +81,7 @@ func (e *StringNotLikeEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type StringEqualsIgnoreCaseEvaluator struct{}
|
||||
|
||||
func (e *StringEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.EqualFold(expected, contextValue) {
|
||||
@@ -240,7 +96,7 @@ func (e *StringEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, c
|
||||
type StringNotEqualsIgnoreCaseEvaluator struct{}
|
||||
|
||||
func (e *StringNotEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.EqualFold(expected, contextValue) {
|
||||
@@ -255,7 +111,7 @@ func (e *StringNotEqualsIgnoreCaseEvaluator) Evaluate(conditionValue interface{}
|
||||
type NumericEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -278,7 +134,7 @@ func (e *NumericEqualsEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type NumericNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -301,7 +157,7 @@ func (e *NumericNotEqualsEvaluator) Evaluate(conditionValue interface{}, context
|
||||
type NumericLessThanEvaluator struct{}
|
||||
|
||||
func (e *NumericLessThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -324,7 +180,7 @@ func (e *NumericLessThanEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type NumericLessThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -347,7 +203,7 @@ func (e *NumericLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, co
|
||||
type NumericGreaterThanEvaluator struct{}
|
||||
|
||||
func (e *NumericGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -370,7 +226,7 @@ func (e *NumericGreaterThanEvaluator) Evaluate(conditionValue interface{}, conte
|
||||
type NumericGreaterThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *NumericGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedFloat, err := strconv.ParseFloat(expected, 64)
|
||||
if err != nil {
|
||||
@@ -393,7 +249,7 @@ func (e *NumericGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{},
|
||||
type DateEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -416,7 +272,7 @@ func (e *DateEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type DateNotEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -439,7 +295,7 @@ func (e *DateNotEqualsEvaluator) Evaluate(conditionValue interface{}, contextVal
|
||||
type DateLessThanEvaluator struct{}
|
||||
|
||||
func (e *DateLessThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -462,7 +318,7 @@ func (e *DateLessThanEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type DateLessThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -485,7 +341,7 @@ func (e *DateLessThanEqualsEvaluator) Evaluate(conditionValue interface{}, conte
|
||||
type DateGreaterThanEvaluator struct{}
|
||||
|
||||
func (e *DateGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -508,7 +364,7 @@ func (e *DateGreaterThanEvaluator) Evaluate(conditionValue interface{}, contextV
|
||||
type DateGreaterThanEqualsEvaluator struct{}
|
||||
|
||||
func (e *DateGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedTime, err := time.Parse(time.RFC3339, expected)
|
||||
if err != nil {
|
||||
@@ -531,7 +387,7 @@ func (e *DateGreaterThanEqualsEvaluator) Evaluate(conditionValue interface{}, co
|
||||
type BoolEvaluator struct{}
|
||||
|
||||
func (e *BoolEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if strings.ToLower(expected) == strings.ToLower(contextValue) {
|
||||
@@ -546,7 +402,7 @@ func (e *BoolEvaluator) Evaluate(conditionValue interface{}, contextValues []str
|
||||
type IpAddressEvaluator struct{}
|
||||
|
||||
func (e *IpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
_, expectedNet, err := net.ParseCIDR(expected)
|
||||
if err != nil {
|
||||
@@ -587,7 +443,7 @@ func (e *IpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type NotIpAddressEvaluator struct{}
|
||||
|
||||
func (e *NotIpAddressEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
_, expectedNet, err := net.ParseCIDR(expected)
|
||||
if err != nil {
|
||||
@@ -628,7 +484,7 @@ func (e *NotIpAddressEvaluator) Evaluate(conditionValue interface{}, contextValu
|
||||
type ArnEqualsEvaluator struct{}
|
||||
|
||||
func (e *ArnEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
for _, contextValue := range contextValues {
|
||||
if expected == contextValue {
|
||||
@@ -643,7 +499,7 @@ func (e *ArnEqualsEvaluator) Evaluate(conditionValue interface{}, contextValues
|
||||
type ArnLikeEvaluator struct{}
|
||||
|
||||
func (e *ArnLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
patterns := getCachedNormalizedValues(conditionValue)
|
||||
patterns := normalizeToStringSlice(conditionValue)
|
||||
for _, pattern := range patterns {
|
||||
for _, contextValue := range contextValues {
|
||||
if wildcard.MatchesWildcard(pattern, contextValue) {
|
||||
@@ -658,7 +514,7 @@ func (e *ArnLikeEvaluator) Evaluate(conditionValue interface{}, contextValues []
|
||||
type NullEvaluator struct{}
|
||||
|
||||
func (e *NullEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool {
|
||||
expectedValues := getCachedNormalizedValues(conditionValue)
|
||||
expectedValues := normalizeToStringSlice(conditionValue)
|
||||
for _, expected := range expectedValues {
|
||||
expectedBool := strings.ToLower(expected) == "true"
|
||||
contextExists := len(contextValues) > 0
|
||||
@@ -851,8 +707,6 @@ func evaluateConditions(conditions PolicyConditions, contextValues map[string][]
|
||||
substitutedValues[i] = SubstituteVariables(v, contextValues, claims)
|
||||
}
|
||||
|
||||
// Pass substituted values (casted to interface{} to match signature if needed, or update evaluators to accept []string)
|
||||
// The evaluators take interface{}, but getCachedNormalizedValues handles []string.
|
||||
if !conditionEvaluator.Evaluate(substitutedValues, contextVals) {
|
||||
return false // If any condition fails, the whole condition block fails
|
||||
}
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package policy_engine
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestConditionValuesLookingAlikeStayDistinct(t *testing.T) {
|
||||
key := "aws:username"
|
||||
list := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("distinct-a", "distinct-b")}}
|
||||
spaced := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("distinct-a distinct-b")}}
|
||||
|
||||
if !EvaluateConditions(list, map[string][]string{key: {"distinct-a"}}, nil, nil) {
|
||||
t.Fatal("list condition must match one of its values")
|
||||
}
|
||||
if EvaluateConditions(spaced, map[string][]string{key: {"distinct-a"}}, nil, nil) {
|
||||
t.Error(`"distinct-a distinct-b" matched "distinct-a"`)
|
||||
}
|
||||
if !EvaluateConditions(spaced, map[string][]string{key: {"distinct-a distinct-b"}}, nil, nil) {
|
||||
t.Error(`"distinct-a distinct-b" did not match itself`)
|
||||
}
|
||||
|
||||
empty := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice()}}
|
||||
blank := PolicyConditions{"StringEquals": {key: NewStringOrStringSlice("")}}
|
||||
if EvaluateConditions(empty, map[string][]string{key: {""}}, nil, nil) {
|
||||
t.Error("empty value list matched")
|
||||
}
|
||||
if !EvaluateConditions(blank, map[string][]string{key: {""}}, nil, nil) {
|
||||
t.Error(`"" did not match itself after an empty value list`)
|
||||
}
|
||||
|
||||
eval := &StringEqualsEvaluator{}
|
||||
if !eval.Evaluate([]interface{}{"json-a", "json-b"}, []string{"json-a"}) {
|
||||
t.Fatal("JSON list must match one of its values")
|
||||
}
|
||||
if eval.Evaluate([]interface{}{"json-a json-b"}, []string{"json-a"}) {
|
||||
t.Error(`JSON "json-a json-b" matched "json-a"`)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketPolicyConditionValuesDoNotLeakAcrossBuckets(t *testing.T) {
|
||||
engine := NewPolicyEngine()
|
||||
policy := func(bucket, effect, key, value string) string {
|
||||
return fmt.Sprintf(`{"Version":"2012-10-17","Statement":[{"Effect":%q,"Principal":"*",
|
||||
"Action":"s3:PutObject","Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"StringEquals":{%q:%s}}}]}`, effect, bucket, key, value)
|
||||
}
|
||||
if err := engine.SetBucketPolicy("victim", policy("victim", "Deny", "s3:x-amz-acl", `["public-read","public-read-write"]`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := engine.SetBucketPolicy("other", policy("other", "Allow", "aws:UserAgent", `"public-read public-read-write"`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
put := func(bucket string, headers map[string]string) PolicyEvaluationResult {
|
||||
r := httptest.NewRequest(http.MethodPut, "/"+bucket+"/key", nil)
|
||||
for k, v := range headers {
|
||||
r.Header.Set(k, v)
|
||||
}
|
||||
return engine.EvaluatePolicyForRequest(bucket, "key", "PutObject", "*", r)
|
||||
}
|
||||
|
||||
if got := put("other", map[string]string{"User-Agent": "public-read public-read-write"}); got != PolicyResultAllow {
|
||||
t.Fatalf("other bucket: got %v, want Allow", got)
|
||||
}
|
||||
if got := put("victim", map[string]string{"x-amz-acl": "public-read"}); got != PolicyResultDeny {
|
||||
t.Errorf("victim bucket: got %v, want Deny", got)
|
||||
}
|
||||
}
|
||||
@@ -623,6 +623,9 @@ func compilePattern(pattern string) (*regexp.Regexp, error) {
|
||||
|
||||
// normalizeToStringSlice converts various types to string slice - kept for backward compatibility
|
||||
func normalizeToStringSlice(value interface{}) []string {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
result, err := normalizeToStringSliceWithError(value)
|
||||
if err != nil {
|
||||
glog.Warningf("unexpected type for policy value: %T, error: %v", value, err)
|
||||
|
||||
@@ -82,6 +82,7 @@ const (
|
||||
AmzObjectLockMode = "X-Amz-Object-Lock-Mode"
|
||||
AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date"
|
||||
AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold"
|
||||
AmzBypassGovernanceRetention = "X-Amz-Bypass-Governance-Retention"
|
||||
|
||||
// S3 checksum headers
|
||||
AmzChecksumAlgorithm = "X-Amz-Checksum-Algorithm"
|
||||
|
||||
@@ -138,7 +138,7 @@ func TestPutObjectPartSkipsBucketDefaultEncryption(t *testing.T) {
|
||||
})
|
||||
|
||||
filePath := "/buckets/b/.uploads/up1/0002_deadbeef.part"
|
||||
r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/o?partNumber=2&uploadId=up1", nil))
|
||||
r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/o?partNumber=2&uploadId=up1", nil), s3a.uploadFiler("b", "o"))
|
||||
etag, code, sseMeta := s3a.putToFiler(r, filePath, strings.NewReader("part data"), "b", "", 2, 0, nil, false, "")
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("putToFiler = %v, want %v", code, s3err.ErrNone)
|
||||
|
||||
@@ -17,7 +17,7 @@ type BucketPolicyEngine struct {
|
||||
// "aws:kms") that was stored when a multipart upload was initiated.
|
||||
// Returns "" when the upload had no SSE or when the entry cannot be found.
|
||||
// Set by S3ApiServer after construction to give the engine filer access.
|
||||
MultipartSSELookup func(bucket, uploadID string) string
|
||||
MultipartSSELookup func(bucket, object, uploadID string) string
|
||||
}
|
||||
|
||||
// NewBucketPolicyEngine creates a new bucket policy engine
|
||||
@@ -169,7 +169,7 @@ func (bpe *BucketPolicyEngine) EvaluatePolicy(bucket, object, action, principal
|
||||
// policy conditions on s3:x-amz-server-side-encryption evaluate correctly.
|
||||
if policy_engine.IsMultipartContinuationAction(s3Action) && bpe.MultipartSSELookup != nil {
|
||||
if uploadID := r.URL.Query().Get("uploadId"); uploadID != "" {
|
||||
args.InheritedSSEAlgorithm = bpe.MultipartSSELookup(bucket, uploadID)
|
||||
args.InheritedSSEAlgorithm = bpe.MultipartSSELookup(bucket, object, uploadID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,12 +65,12 @@ func TestCopyAfterRoutedPut(t *testing.T) {
|
||||
var uploadID string
|
||||
if multipart {
|
||||
uploadID = s3a.generateUploadID("final/object")
|
||||
replica.mu.Lock()
|
||||
replica.entries["/buckets/destination/.uploads/"+uploadID] = &filer_pb.Entry{
|
||||
owner.mu.Lock()
|
||||
owner.entries["/buckets/destination/.uploads/"+uploadID] = &filer_pb.Entry{
|
||||
Name: uploadID, IsDirectory: true,
|
||||
Extended: map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte("final/object")},
|
||||
}
|
||||
replica.mu.Unlock()
|
||||
owner.mu.Unlock()
|
||||
copyURL += "?partNumber=1&uploadId=" + uploadID
|
||||
}
|
||||
copyRequest := httptest.NewRequest(http.MethodPut, copyURL, nil)
|
||||
@@ -87,15 +87,17 @@ func TestCopyAfterRoutedPut(t *testing.T) {
|
||||
|
||||
replica.mu.Lock()
|
||||
if multipart {
|
||||
replica.mu.Unlock()
|
||||
owner.mu.Lock()
|
||||
var part *filer_pb.Entry
|
||||
for key, entry := range replica.entries {
|
||||
for key, entry := range owner.entries {
|
||||
if strings.HasPrefix(key, "/buckets/destination/.uploads/"+uploadID+"/") && !entry.IsDirectory {
|
||||
part = entry
|
||||
}
|
||||
}
|
||||
require.NotNil(t, part, "copy must persist the part")
|
||||
owner.mu.Unlock()
|
||||
require.NotNil(t, part, "copy must persist the part next to the upload")
|
||||
require.Equal(t, stored.Attributes.Md5, part.Attributes.Md5)
|
||||
replica.mu.Unlock()
|
||||
} else {
|
||||
// The destination commits on its write owner, the same filer a
|
||||
// later copy resolves it from before replication.
|
||||
|
||||
@@ -239,6 +239,18 @@ func TestGovernanceBypassHeader(t *testing.T) {
|
||||
expectedResult: true,
|
||||
description: "Header with 'true' value should enable bypass",
|
||||
},
|
||||
{
|
||||
name: "bypass_header_title_case",
|
||||
headerValue: "True",
|
||||
expectedResult: true,
|
||||
description: "botocore and the AWS CLI send 'True'",
|
||||
},
|
||||
{
|
||||
name: "bypass_header_upper_case",
|
||||
headerValue: "TRUE",
|
||||
expectedResult: true,
|
||||
description: "Header value is case-insensitive",
|
||||
},
|
||||
{
|
||||
name: "bypass_header_false",
|
||||
headerValue: "false",
|
||||
@@ -266,7 +278,7 @@ func TestGovernanceBypassHeader(t *testing.T) {
|
||||
req.Header.Set("x-amz-bypass-governance-retention", tt.headerValue)
|
||||
}
|
||||
|
||||
result := req.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
result := governanceBypassRequested(req)
|
||||
|
||||
if result != tt.expectedResult {
|
||||
t.Errorf("bypass header check = %v, want %v. %s", result, tt.expectedResult, tt.description)
|
||||
@@ -450,7 +462,7 @@ func TestGovernancePermissionHTTPFlow(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test the header processing logic used in handlers
|
||||
bypassGovernance := req.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
bypassGovernance := governanceBypassRequested(req)
|
||||
|
||||
if bypassGovernance != tt.expectedBypassGovernance {
|
||||
t.Errorf("Expected bypassGovernance to be %v, got %v", tt.expectedBypassGovernance, bypassGovernance)
|
||||
@@ -471,7 +483,7 @@ func TestGovernancePermissionMethodCalls(t *testing.T) {
|
||||
// Extract parameters as done in the handler
|
||||
bucket, object := s3_constants.GetBucketAndObject(req)
|
||||
versionId := req.URL.Query().Get("versionId")
|
||||
bypassGovernance := req.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
bypassGovernance := governanceBypassRequested(req)
|
||||
|
||||
// Verify the parameters are extracted correctly
|
||||
// Note: The actual bucket and object extraction depends on the URL structure
|
||||
@@ -491,7 +503,7 @@ func TestGovernancePermissionMethodCalls(t *testing.T) {
|
||||
|
||||
// Extract parameters as done in the handler
|
||||
bucket, object := s3_constants.GetBucketAndObject(req)
|
||||
bypassGovernance := req.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
bypassGovernance := governanceBypassRequested(req)
|
||||
versioningEnabled := true // Would be determined by isVersioningEnabled(bucket)
|
||||
|
||||
// Verify the parameters are extracted correctly
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
// LifecycleDelete executes one (rule, action) verdict: re-fetch, identity
|
||||
@@ -88,44 +89,40 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
return resp, nil
|
||||
}
|
||||
}
|
||||
// Past the attempt deadline the worker may already be resending
|
||||
// this request, so a late marker write would stack a second one.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch state {
|
||||
case s3_constants.VersioningEnabled:
|
||||
if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil {
|
||||
return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil
|
||||
written, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry))
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !written {
|
||||
return noopResolved("LATEST_IS_DELETE_MARKER"), nil
|
||||
}
|
||||
return done(), nil
|
||||
case s3_constants.VersioningSuspended:
|
||||
// Best-effort null delete; NotFound is benign. Route it when an
|
||||
// owner resolves — on the default filer an absent entry deletes as
|
||||
// success, and a null version that only reached the owner would
|
||||
// linger. Lifecycle never bypasses retention.
|
||||
nullDeleted := false
|
||||
if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" {
|
||||
worm, lockErr := s3a.isObjectLockEnabled(req.Bucket)
|
||||
if lockErr == nil {
|
||||
code, settled := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, nil)
|
||||
if settled {
|
||||
if code == s3err.ErrAccessDenied {
|
||||
return retryLater("ACCESS_DENIED: deleteNullVersion: object lock"), nil
|
||||
}
|
||||
if code != s3err.ErrNone {
|
||||
return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code), nil
|
||||
}
|
||||
nullDeleted = true
|
||||
}
|
||||
}
|
||||
// Marker first: the pointer outranks the null object, so the key
|
||||
// reads as deleted once it lands, and a null delete that fails
|
||||
// after it leaves a noncurrent version instead of exposing an
|
||||
// older one to a retry that then finds no null to expire.
|
||||
nullEntry, resp := s3a.lifecycleNullVersion(req, entry)
|
||||
if resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
if !nullDeleted {
|
||||
if err := s3a.deleteSpecificObjectVersion(ctx, req.Bucket, req.ObjectPath, "null", metadataOnly); err != nil {
|
||||
if !errors.Is(err, filer_pb.ErrNotFound) && !errors.Is(err, ErrVersionNotFound) {
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + err.Error()), nil
|
||||
}
|
||||
}
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err := s3a.createDeleteMarker(req.Bucket, req.ObjectPath); err != nil {
|
||||
return retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error()), nil
|
||||
if _, resp := s3a.lifecycleDeleteMarker(req.Bucket, req.ObjectPath, lifecycleAddressedVersion(req, entry)); resp != nil {
|
||||
return resp, nil
|
||||
}
|
||||
return done(), nil
|
||||
if nullEntry == nil {
|
||||
return done(), nil
|
||||
}
|
||||
return s3a.lifecycleDeleteNullVersion(req, nullEntry, metadataOnly), nil
|
||||
default:
|
||||
err := s3a.WithFilerClient(false, func(c filer_pb.SeaweedFilerClient) error {
|
||||
return s3a.deleteUnversionedObjectWithClient(ctx, c, req.Bucket, req.ObjectPath, metadataOnly)
|
||||
@@ -225,6 +222,147 @@ func (s3a *S3ApiServer) lifecycleDispatch(ctx context.Context, req *s3_lifecycle
|
||||
}
|
||||
}
|
||||
|
||||
// lifecycleDeleteMarker writes an expiration's delete marker as a
|
||||
// compare-and-set on the .versions pointer, skipping it when the latest is
|
||||
// already a delete marker. An attempt the worker gave up on may still be
|
||||
// running while its retry writes, so whichever commits second sees the marker.
|
||||
// A non-empty expected is the version that passed the expiration checks; a
|
||||
// different latest resolves the request as stale instead of hiding it.
|
||||
func (s3a *S3ApiServer) lifecycleDeleteMarker(bucket, object, expected string) (written bool, resp *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
normalized := s3_constants.NormalizeObjectKey(object)
|
||||
if s3a.objectWriteOwner(bucket, normalized) != "" {
|
||||
return s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected)
|
||||
}
|
||||
// Off the ring the marker commits without the guard, so the pointer read
|
||||
// and the write serialize with other writers on the object write lock.
|
||||
code := s3a.withObjectWriteLock(bucket, normalized, nil, func() s3err.ErrorCode {
|
||||
written, resp = s3a.lifecycleDeleteMarkerIf(bucket, object, normalized, expected)
|
||||
return s3err.ErrNone
|
||||
})
|
||||
if code != s3err.ErrNone {
|
||||
return false, retryLater("TRANSPORT_ERROR: object write lock: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
return written, resp
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) lifecycleDeleteMarkerIf(bucket, object, normalized, expected string) (bool, *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
for range 3 {
|
||||
pointer, err := s3a.getObjectScopedEntryRoutedByKey(bucket, normalized, s3a.bucketDir(bucket), normalized+s3_constants.VersionsFolder)
|
||||
if err != nil {
|
||||
if !isObjectEntryMiss(err) {
|
||||
return false, retryLater("TRANSPORT_ERROR: latest-pointer lookup: " + err.Error())
|
||||
}
|
||||
pointer = nil
|
||||
}
|
||||
if pointer != nil && string(pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker]) == "true" && !nullVersionIsLatest(pointer) {
|
||||
return false, nil
|
||||
}
|
||||
if expected != "" && pointer != nil {
|
||||
if latest := pointerLatestVersion(pointer); latest != "" && latest != expected {
|
||||
return false, noopResolved("VERSION_NOT_LATEST")
|
||||
}
|
||||
}
|
||||
guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
|
||||
Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL,
|
||||
ExpectedEntry: pointer,
|
||||
}}}
|
||||
if _, err := s3a.createDeleteMarkerIf(bucket, object, guard); err != nil {
|
||||
if errors.Is(err, errLatestVersionChanged) {
|
||||
continue
|
||||
}
|
||||
return false, retryLater("TRANSPORT_ERROR: createDeleteMarker: " + err.Error())
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
return false, retryLater("LATEST_CHANGED: createDeleteMarker")
|
||||
}
|
||||
|
||||
// lifecycleNullVersion reads the null object the suspended expiration will
|
||||
// delete. When it is the version validated for expiration, a different or
|
||||
// missing entry resolves the request as stale.
|
||||
func (s3a *S3ApiServer) lifecycleNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, validated *filer_pb.Entry) (*filer_pb.Entry, *s3_lifecycle_pb.LifecycleDeleteResponse) {
|
||||
nullEntry, err := s3a.getObjectEntryRoutedByKey(req.Bucket, req.ObjectPath)
|
||||
if err != nil && !isObjectEntryMiss(err) {
|
||||
return nil, retryLater("TRANSPORT_ERROR: null version lookup: " + err.Error())
|
||||
}
|
||||
if err != nil || !isNullVersionObject(nullEntry, req.ObjectPath) {
|
||||
nullEntry = nil
|
||||
}
|
||||
if lifecycleAddressedVersion(req, validated) == "null" && (nullEntry == nil || !identityMatches(computeEntryIdentity(nullEntry), computeEntryIdentity(validated))) {
|
||||
return nil, noopResolved("STALE_IDENTITY")
|
||||
}
|
||||
return nullEntry, nil
|
||||
}
|
||||
|
||||
// lifecycleDeleteNullVersion deletes the null version once the marker hides
|
||||
// it, even past the attempt deadline. Only the entry read before the marker is
|
||||
// deleted: a null object written since is newer and stays. Lifecycle never
|
||||
// bypasses retention.
|
||||
func (s3a *S3ApiServer) lifecycleDeleteNullVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, nullEntry *filer_pb.Entry, metadataOnly bool) *s3_lifecycle_pb.LifecycleDeleteResponse {
|
||||
if owner := s3a.routableWriteOwner(req.Bucket, req.ObjectPath); owner != "" {
|
||||
worm, lockErr := s3a.isObjectLockEnabled(req.Bucket)
|
||||
if lockErr != nil {
|
||||
return retryLater("TRANSPORT_ERROR: object lock lookup: " + lockErr.Error())
|
||||
}
|
||||
guard := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{
|
||||
Kind: filer_pb.WriteCondition_IF_ENTRY_EQUAL,
|
||||
ExpectedEntry: nullEntry,
|
||||
}}}
|
||||
switch code, _ := s3a.routedDeleteNullVersion(owner, req.Bucket, req.ObjectPath, worm, false, metadataOnly, guard); code {
|
||||
case s3err.ErrNone, s3err.ErrPreconditionFailed:
|
||||
return done()
|
||||
case s3err.ErrAccessDenied:
|
||||
return retryLater("ACCESS_DENIED: deleteNullVersion: object lock")
|
||||
default:
|
||||
return retryLater("TRANSPORT_ERROR: routedDeleteNullVersion: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
}
|
||||
// Off the ring, writers serialize on the object write lock instead.
|
||||
dir, name := util.NewFullPath(s3a.bucketDir(req.Bucket), req.ObjectPath).DirAndName()
|
||||
var opErr error
|
||||
code := s3a.withObjectWriteLock(req.Bucket, req.ObjectPath, func() s3err.ErrorCode {
|
||||
current, err := s3a.getEntry(dir, name)
|
||||
switch {
|
||||
case errors.Is(err, filer_pb.ErrNotFound):
|
||||
return s3err.ErrPreconditionFailed
|
||||
case err != nil:
|
||||
opErr = err
|
||||
return s3err.ErrInternalError
|
||||
case !identityMatches(computeEntryIdentity(current), computeEntryIdentity(nullEntry)):
|
||||
return s3err.ErrPreconditionFailed
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}, func() s3err.ErrorCode {
|
||||
if opErr = s3a.deleteSpecificObjectVersion(context.Background(), req.Bucket, req.ObjectPath, "null", metadataOnly); opErr != nil {
|
||||
return s3err.ErrInternalError
|
||||
}
|
||||
return s3err.ErrNone
|
||||
})
|
||||
switch {
|
||||
case code == s3err.ErrNone || code == s3err.ErrPreconditionFailed:
|
||||
return done()
|
||||
case opErr != nil:
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + opErr.Error())
|
||||
default:
|
||||
return retryLater("TRANSPORT_ERROR: deleteNullVersion: " + s3err.GetAPIError(code).Code)
|
||||
}
|
||||
}
|
||||
|
||||
// pointerLatestVersion reads the latest version off a .versions pointer the
|
||||
// way getLatestObjectVersion does: a named pointer with its file outranks the
|
||||
// null object, the null-is-latest signal names the null version, and anything
|
||||
// else leaves the latest to the caller's earlier read.
|
||||
func pointerLatestVersion(pointer *filer_pb.Entry) string {
|
||||
id := string(pointer.Extended[s3_constants.ExtLatestVersionIdKey])
|
||||
if id != "" && len(pointer.Extended[s3_constants.ExtLatestVersionFileNameKey]) > 0 {
|
||||
return id
|
||||
}
|
||||
if nullVersionIsLatest(pointer) {
|
||||
return "null"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// lifecycleRequireLatest refuses a current-version expiration whose version
|
||||
// is no longer the key's latest. A version file never changes, so the identity
|
||||
// check cannot see a newer write.
|
||||
@@ -236,22 +374,26 @@ func (s3a *S3ApiServer) lifecycleRequireLatest(req *s3_lifecycle_pb.LifecycleDel
|
||||
}
|
||||
return retryLater("TRANSPORT_ERROR: latest-version lookup: " + err.Error())
|
||||
}
|
||||
versionIdOrNull := func(e *filer_pb.Entry) string {
|
||||
if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" {
|
||||
return id
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
addressed := req.VersionId
|
||||
if addressed == "" {
|
||||
addressed = versionIdOrNull(entry)
|
||||
}
|
||||
if versionIdOrNull(latest) != addressed {
|
||||
if versionIdOrNull(latest) != lifecycleAddressedVersion(req, entry) {
|
||||
return noopResolved("VERSION_NOT_LATEST")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func versionIdOrNull(e *filer_pb.Entry) string {
|
||||
if id := string(e.Extended[s3_constants.ExtVersionIdKey]); id != "" {
|
||||
return id
|
||||
}
|
||||
return "null"
|
||||
}
|
||||
|
||||
func lifecycleAddressedVersion(req *s3_lifecycle_pb.LifecycleDeleteRequest, entry *filer_pb.Entry) string {
|
||||
if req.VersionId != "" {
|
||||
return req.VersionId
|
||||
}
|
||||
return versionIdOrNull(entry)
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) lifecycleAbortMPU(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
// req.ObjectPath is `.uploads/<upload_id>` (set by the router from the
|
||||
// init directory's bucket-relative path); reject anything that isn't
|
||||
|
||||
@@ -3,7 +3,10 @@ package s3api
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -503,6 +506,690 @@ func TestLifecycleDelete_CurrentExpirationRequiresLatestVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_ExpiredAttemptWritesNoMarker(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
for _, versioning := range []string{s3_constants.VersioningEnabled, s3_constants.VersioningSuspended} {
|
||||
t.Run(versioning, func(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}
|
||||
fake := &ambiguousPutFiler{entries: entries, apply: true}
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, fake))
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: versioning})
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
cancel()
|
||||
_, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker")
|
||||
require.Len(t, fake.entries, 2)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// suspendedNullServer serves a suspended bucket whose latest is the null
|
||||
// object, with an older version that a lost delete marker would expose.
|
||||
func suspendedNullServer(t *testing.T, routed bool) (*S3ApiServer, *ambiguousPutFiler, *filer_pb.Entry) {
|
||||
olderId := generateVersionId(true)
|
||||
older := &filer_pb.Entry{
|
||||
Name: "v_" + olderId,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_600_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(olderId)},
|
||||
}
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
}
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}},
|
||||
"/buckets/b/obj.versions/" + older.Name: proto.Clone(older).(*filer_pb.Entry),
|
||||
}}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
|
||||
if routed {
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
}
|
||||
return s3a, fake, null
|
||||
}
|
||||
|
||||
func isMarkerTxn(req *filer_pb.ObjectTransactionRequest) bool {
|
||||
for _, m := range req.Mutations {
|
||||
if m.Type == filer_pb.ObjectMutation_PUT {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func requireLatestIsMarker(t *testing.T, s3a *S3ApiServer) {
|
||||
t.Helper()
|
||||
got, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, "true", string(got.Extended[s3_constants.ExtDeleteMarkerKey]), "the older version must stay hidden behind the marker")
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringMarkerStillDeletesNull(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
ctx, cancel := context.WithCancel(s3LocalPeerCtx())
|
||||
defer cancel()
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { cancel() }
|
||||
resp, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, 2, fake.txnCount, "the marker and the null delete")
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_DeadlineDuringNullReadWritesNoMarker(t *testing.T) {
|
||||
nullReads := func(cancelAt int, cancel func()) int {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
reads := 0
|
||||
fake.beforeLookup = func(key string) {
|
||||
if key != "/buckets/b/obj" || fake.txnCount > 0 {
|
||||
return
|
||||
}
|
||||
if reads++; reads == cancelAt {
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
ctx, stop := context.WithCancel(s3LocalPeerCtx())
|
||||
defer stop()
|
||||
if cancel == nil {
|
||||
cancel = stop
|
||||
}
|
||||
_, err := s3a.LifecycleDelete(ctx, &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
if cancelAt > 0 {
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
require.Zero(t, fake.txnCount, "an attempt past its deadline must not write a delete marker")
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
}
|
||||
return reads
|
||||
}
|
||||
last := nullReads(0, nil)
|
||||
require.Positive(t, last)
|
||||
nullReads(last, nil)
|
||||
}
|
||||
|
||||
// A marker write that fails leaves the null version in place, so the retry
|
||||
// still finds it latest and finishes the expiration.
|
||||
func TestLifecycleDelete_SuspendedMarkerFailureLeavesNullForRetry(t *testing.T) {
|
||||
for _, routed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, routed)
|
||||
failMarker := true
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
fake.txnNoApply = failMarker && isMarkerTxn(req)
|
||||
fake.txnErr = nil
|
||||
if fake.txnNoApply {
|
||||
fake.txnErr = errors.New("owner unavailable")
|
||||
}
|
||||
}
|
||||
fake.createErr = errors.New("filer unavailable")
|
||||
req := &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj", "the null version must outlive a failed marker")
|
||||
|
||||
failMarker, fake.createErr = false, nil
|
||||
resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A routed null delete that fails retries rather than deleting through the
|
||||
// default filer, which may not have the owner's newer null object yet.
|
||||
func TestLifecycleDelete_RoutedNullDeleteFailureRetries(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
fake.txnNoApply = !isMarkerTxn(req)
|
||||
fake.txnErr = nil
|
||||
if fake.txnNoApply {
|
||||
fake.txnErr = errors.New("owner unavailable")
|
||||
}
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
requireLatestIsMarker(t, s3a)
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_RoutedNullDeleteObjectLockLookupFailureRetries(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, true)
|
||||
fake.txnCheck = func(req *filer_pb.ObjectTransactionRequest) {
|
||||
if isMarkerTxn(req) {
|
||||
s3a.bucketConfigCache.Remove("b")
|
||||
fake.lookupErr, fake.lookupFailKey = status.Error(codes.Unavailable, "filer unavailable"), "/buckets/b"
|
||||
}
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_RETRY_LATER, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, 1, fake.txnCount, "only the marker")
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
}
|
||||
|
||||
// A null object written after the expiration read the old one is not deleted.
|
||||
func TestLifecycleDelete_SuspendedNullRewrittenBeforeDeleteSurvives(t *testing.T) {
|
||||
for _, routed := range []bool{true, false} {
|
||||
t.Run(fmt.Sprintf("routed=%v", routed), func(t *testing.T) {
|
||||
s3a, fake, null := suspendedNullServer(t, routed)
|
||||
rewritten := proto.Clone(null).(*filer_pb.Entry)
|
||||
rewritten.Attributes.Mtime++
|
||||
rewritten.Chunks = []*filer_pb.FileChunk{{FileId: "3,03637037d6", Size: 3}}
|
||||
rewrite := func() { fake.entries["/buckets/b/obj"] = proto.Clone(rewritten).(*filer_pb.Entry) }
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount == 1 {
|
||||
rewrite()
|
||||
}
|
||||
}
|
||||
fake.beforeCreate = rewrite
|
||||
var locked []string
|
||||
s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) {
|
||||
locked = append(locked, bucket+"/"+object)
|
||||
return noopObjectWriteLock{}, nil
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
require.Equal(t, "3,03637037d6", fake.entries["/buckets/b/obj"].Chunks[0].GetFileIdString())
|
||||
if !routed {
|
||||
require.Equal(t, []string{"b/obj", "b/obj"}, locked, "the marker, then the recheck and the delete, must hold the object write lock")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A null PUT that lands after a suspended expiration validated a named version
|
||||
// is neither hidden nor deleted, whether it lands before the null read or
|
||||
// before the routed marker commit.
|
||||
func TestLifecycleDelete_SuspendedPutOverNamedVersionSurvives(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fresh := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
routed bool
|
||||
atRead bool
|
||||
}{
|
||||
{"routed/at-null-read", true, true},
|
||||
{"routed/at-marker", true, false},
|
||||
{"unrouted/at-null-read", false, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
put := func() {
|
||||
if _, ok := fake.entries["/buckets/b/obj"]; ok {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true")
|
||||
}
|
||||
if tc.atRead {
|
||||
fake.beforeLookup = func(key string) {
|
||||
if key == "/buckets/b/obj" {
|
||||
put()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) { put() }
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningSuspended})
|
||||
if tc.routed {
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj")
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A timed-out attempt can still be running when its retry writes the marker;
|
||||
// the stale attempt's commit then finds the pointer moved and adds no second one.
|
||||
func TestLifecycleDelete_OverlappingAttemptWritesOneMarker(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
retryMarker := &filer_pb.Entry{
|
||||
Name: "v_retry",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtVersionIdKey: []byte("retry"),
|
||||
s3_constants.ExtDeleteMarkerKey: []byte("true"),
|
||||
},
|
||||
}
|
||||
cases := []struct {
|
||||
versioning string
|
||||
markerTxn int
|
||||
outcome s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
}{
|
||||
{s3_constants.VersioningEnabled, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{s3_constants.VersioningSuspended, 1, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.versioning, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != tc.markerTxn {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_retry"] = proto.Clone(retryMarker).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("retry")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_retry")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIsDeleteMarker] = []byte("true")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: tc.versioning})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, tc.outcome, resp.Outcome, resp.Reason)
|
||||
var markers []string
|
||||
for key, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers = append(markers, key)
|
||||
}
|
||||
}
|
||||
require.Equal(t, []string{"/buckets/b/obj.versions/v_retry"}, markers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// A version promoted after the expiration checks passed must not be hidden by
|
||||
// the marker: the conflict retry finds a different latest and resolves stale.
|
||||
func TestLifecycleDelete_VersionPromotedBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
latest := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
promoted := &filer_pb.Entry{
|
||||
Name: "v_new",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")},
|
||||
}
|
||||
for _, versionId := range []string{"old", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(latest).(*filer_pb.Entry),
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(latest),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_VersionPromotedOverNullBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
}
|
||||
promoted := &filer_pb.Entry{
|
||||
Name: "v_new",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("new")},
|
||||
}
|
||||
for _, versionId := range []string{"null", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtNullVersionIsLatestKey: []byte("true"),
|
||||
}},
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj.versions/v_new"] = proto.Clone(promoted).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtNullVersionIsLatestKey)
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte("new")
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_new")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_SuspendedPutOverVersionBeforeMarkerIsNotHidden(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fresh := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
}
|
||||
for _, versionId := range []string{"old", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
fake.txnCheck = func(*filer_pb.ObjectTransactionRequest) {
|
||||
if fake.txnCount != 1 {
|
||||
return
|
||||
}
|
||||
fake.entries["/buckets/b/obj"] = proto.Clone(fresh).(*filer_pb.Entry)
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIdKey)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionFileNameKey)
|
||||
pointer.Extended[s3_constants.ExtNullVersionIsLatestKey] = []byte("true")
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED, resp.Outcome, resp.Reason)
|
||||
require.Equal(t, "VERSION_NOT_LATEST", resp.Reason)
|
||||
for key, e := range fake.entries {
|
||||
require.NotEqual(t, "true", string(e.Extended[s3_constants.ExtDeleteMarkerKey]), key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLifecycleDelete_NullUnderIdOnlyPointerExpires(t *testing.T) {
|
||||
null := &filer_pb.Entry{
|
||||
Name: "obj",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
}
|
||||
for _, versionId := range []string{"null", ""} {
|
||||
t.Run("version="+versionId, func(t *testing.T) {
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj": proto.Clone(null).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
}},
|
||||
}}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: versionId,
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(null),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
markers := 0
|
||||
for _, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers++
|
||||
}
|
||||
}
|
||||
require.Equal(t, 1, markers)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type releaseObjectWriteLock func()
|
||||
|
||||
func (r releaseObjectWriteLock) StopShortLivedLock() error {
|
||||
r()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Off the ring a PUT that commits between the marker's pointer read and its
|
||||
// write would be hidden; holding the object write lock makes it wait and land
|
||||
// after the marker instead.
|
||||
func TestLifecycleDelete_UnroutedMarkerHoldsObjectWriteLock(t *testing.T) {
|
||||
old := &filer_pb.Entry{
|
||||
Name: "v_old",
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_000, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,01637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("old")},
|
||||
}
|
||||
fake := &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions/v_old": proto.Clone(old).(*filer_pb.Entry),
|
||||
"/buckets/b/obj.versions": {Name: "obj" + s3_constants.VersionsFolder, IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}, Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte("old"),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_old"),
|
||||
}},
|
||||
}}
|
||||
interleavedId := generateVersionId(true)
|
||||
put := func(id string) {
|
||||
fake.entries["/buckets/b/obj.versions/v_"+id] = &filer_pb.Entry{
|
||||
Name: "v_" + id,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: 1_700_000_100, FileSize: 5},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,02637037d6", Size: 5}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)},
|
||||
}
|
||||
pointer := fake.entries["/buckets/b/obj.versions"]
|
||||
pointer.Extended[s3_constants.ExtLatestVersionIdKey] = []byte(id)
|
||||
pointer.Extended[s3_constants.ExtLatestVersionFileNameKey] = []byte("v_" + id)
|
||||
delete(pointer.Extended, s3_constants.ExtLatestVersionIsDeleteMarker)
|
||||
}
|
||||
held, waiting, putDone := false, false, false
|
||||
fake.beforeCreate = func() {
|
||||
switch {
|
||||
case putDone || waiting:
|
||||
case held:
|
||||
waiting = true
|
||||
default:
|
||||
put(interleavedId)
|
||||
putDone = true
|
||||
}
|
||||
}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled})
|
||||
s3a.newObjectWriteLock = func(bucket, object string) (objectWriteLock, error) {
|
||||
held = true
|
||||
return releaseObjectWriteLock(func() {
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
held = false
|
||||
if waiting {
|
||||
put(generateVersionId(true))
|
||||
putDone = true
|
||||
}
|
||||
}), nil
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: "old",
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS,
|
||||
ExpectedIdentity: computeEntryIdentity(old),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, resp.Outcome, resp.Reason)
|
||||
require.True(t, putDone)
|
||||
latest, err := s3a.getLatestObjectVersion("b", "obj")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, "true", string(latest.Extended[s3_constants.ExtDeleteMarkerKey]), "the PUT must not be hidden by the marker")
|
||||
}
|
||||
|
||||
// Expired-marker cleanup removes a marker that a lock was recorded on: markers
|
||||
// are never WORM-protected, so the routed delete carries no WORM guard.
|
||||
func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) {
|
||||
@@ -552,3 +1239,123 @@ func TestLifecycleDelete_LockedExpiredDeleteMarkerSkipsWormGuard(t *testing.T) {
|
||||
require.Len(t, conditions, 1)
|
||||
require.Nil(t, conditions[0], "marker delete must not carry WORM guards")
|
||||
}
|
||||
|
||||
// A worker retries a request whose reply it lost; the replay must resolve
|
||||
// against what the first attempt already did.
|
||||
func TestLifecycleDelete_ReplayAfterLostReply(t *testing.T) {
|
||||
version := func(id string, mtime int64, extra map[string][]byte) *filer_pb.Entry {
|
||||
e := &filer_pb.Entry{
|
||||
Name: "v_" + id,
|
||||
Attributes: &filer_pb.FuseAttributes{Mtime: mtime, FileSize: 3},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "3,0" + id + "637037d6", Size: 3}},
|
||||
Extended: map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)},
|
||||
}
|
||||
for k, v := range extra {
|
||||
e.Extended[k] = v
|
||||
}
|
||||
return e
|
||||
}
|
||||
versionsDir := func(latestId string) *filer_pb.Entry {
|
||||
return &filer_pb.Entry{
|
||||
Name: "obj" + s3_constants.VersionsFolder,
|
||||
IsDirectory: true,
|
||||
Attributes: &filer_pb.FuseAttributes{},
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtLatestVersionIdKey: []byte(latestId),
|
||||
s3_constants.ExtLatestVersionFileNameKey: []byte("v_" + latestId),
|
||||
},
|
||||
}
|
||||
}
|
||||
locked := map[string][]byte{
|
||||
s3_constants.ExtObjectLockModeKey: []byte(s3_constants.RetentionModeCompliance),
|
||||
s3_constants.ExtRetentionUntilDateKey: []byte(strconv.FormatInt(time.Now().Add(24*time.Hour).Unix(), 10)),
|
||||
}
|
||||
oldVersion := version("1", 1_700_000_000, nil)
|
||||
newVersion := version("2", 1_700_000_100, nil)
|
||||
lockedVersion := version("1", 1_700_000_000, locked)
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
objectLock bool
|
||||
kind s3_lifecycle_pb.ActionKind
|
||||
target *filer_pb.Entry
|
||||
entries map[string]*filer_pb.Entry
|
||||
first s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
replay s3_lifecycle_pb.LifecycleDeleteOutcome
|
||||
}{
|
||||
{"current expiration writes one delete marker", false, s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS, oldVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("1"),
|
||||
"/buckets/b/obj.versions/v_1": oldVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{"noncurrent version", false, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, oldVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("2"),
|
||||
"/buckets/b/obj.versions/v_1": oldVersion,
|
||||
"/buckets/b/obj.versions/v_2": newVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, s3_lifecycle_pb.LifecycleDeleteOutcome_NOOP_RESOLVED},
|
||||
{"object-locked noncurrent version", true, s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS, lockedVersion, map[string]*filer_pb.Entry{
|
||||
"/buckets/b/obj.versions": versionsDir("2"),
|
||||
"/buckets/b/obj.versions/v_1": lockedVersion,
|
||||
"/buckets/b/obj.versions/v_2": newVersion,
|
||||
}, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK, s3_lifecycle_pb.LifecycleDeleteOutcome_SKIPPED_OBJECT_LOCK},
|
||||
}
|
||||
for _, c := range cases {
|
||||
t.Run(c.name, func(t *testing.T) {
|
||||
entries := map[string]*filer_pb.Entry{}
|
||||
for k, v := range c.entries {
|
||||
entries[k] = proto.Clone(v).(*filer_pb.Entry)
|
||||
}
|
||||
fake := &ambiguousPutFiler{entries: entries, apply: true}
|
||||
addr := startFakeFiler(t, fake)
|
||||
s3a := newPutTestServer(t, addr)
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
cfg := &BucketConfig{Name: "b", Versioning: s3_constants.VersioningEnabled}
|
||||
if c.objectLock {
|
||||
cfg.ObjectLockConfig = &ObjectLockConfiguration{ObjectLockEnabled: s3_constants.ObjectLockEnabled}
|
||||
}
|
||||
s3a.bucketConfigCache.Set("b", cfg)
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, addr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{addr}, 1)
|
||||
|
||||
req := &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "b",
|
||||
ObjectPath: "obj",
|
||||
VersionId: string(c.target.Extended[s3_constants.ExtVersionIdKey]),
|
||||
ActionKind: c.kind,
|
||||
ExpectedIdentity: computeEntryIdentity(c.target),
|
||||
}
|
||||
resp, err := s3a.LifecycleDelete(s3LocalPeerCtx(), req)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.first, resp.Outcome, resp.Reason)
|
||||
|
||||
fake.mu.Lock()
|
||||
afterFirst := len(fake.entries)
|
||||
txns := fake.txnCount
|
||||
fake.mu.Unlock()
|
||||
|
||||
resp, err = s3a.LifecycleDelete(s3LocalPeerCtx(), proto.Clone(req).(*s3_lifecycle_pb.LifecycleDeleteRequest))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, c.replay, resp.Outcome, resp.Reason)
|
||||
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
require.Equal(t, txns, fake.txnCount, "the replay must not write")
|
||||
require.Len(t, fake.entries, afterFirst)
|
||||
markers := 0
|
||||
for _, e := range fake.entries {
|
||||
if string(e.Extended[s3_constants.ExtDeleteMarkerKey]) == "true" {
|
||||
markers++
|
||||
}
|
||||
}
|
||||
if c.kind == s3_lifecycle_pb.ActionKind_EXPIRATION_DAYS {
|
||||
require.Equal(t, 1, markers, "a replayed expiration must not stack delete markers")
|
||||
} else {
|
||||
require.Zero(t, markers)
|
||||
}
|
||||
if c.objectLock {
|
||||
require.Contains(t, fake.entries, "/buckets/b/obj.versions/v_1", "a locked version must survive both attempts")
|
||||
} else if c.kind == s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS {
|
||||
require.NotContains(t, fake.entries, "/buckets/b/obj.versions/v_1")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -761,7 +761,7 @@ func (s3a *S3ApiServer) finalizeCopyDestination(dstBucket, dstObject, dstVersion
|
||||
// Route the pointer flip to the owner filer when known (off the
|
||||
// distributed lock); RECOMPUTE_LATEST picks the just-written version.
|
||||
if owner != "" {
|
||||
if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil); code != s3err.ErrNone {
|
||||
if code := s3a.routedVersionedFinalize(owner, dstBucket, normalizedObject, isNewFormatVersionId(versionId), nil, nil); code != s3err.ErrNone {
|
||||
if rollbackErr := s3a.rollbackCopyVersion(owner, dstBucket, normalizedObject, versionObjectPath); rollbackErr != nil {
|
||||
glog.Errorf("CopyObjectHandler: failed to rollback version %s for %s/%s after routed finalize error: %v", versionId, dstBucket, normalizedObject, rollbackErr)
|
||||
}
|
||||
@@ -1181,7 +1181,7 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
// uploadEntry=nil would silently skip the SSE check on the destination
|
||||
// side and could send a plain-source copy through the raw-byte fast
|
||||
// path even though the destination's encryption state is unknown.
|
||||
uploadEntry, uploadEntryErr := s3a.getEntry(s3a.genUploadsFolder(dstBucket), uploadID)
|
||||
uploadEntry, uploadEntryErr := s3a.uploadFiler(dstBucket, dstObject).getEntry(s3a.genUploadsFolder(dstBucket), uploadID)
|
||||
if uploadEntryErr != nil {
|
||||
if errors.Is(uploadEntryErr, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
@@ -1285,14 +1285,15 @@ func (s3a *S3ApiServer) CopyObjectPartHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// Save the part entry to the multipart uploads folder
|
||||
// Check if part exists and remove it first (allow re-copying same part)
|
||||
if exists, _ := s3a.exists(uploadDir, partName, false); exists {
|
||||
if err := s3a.rm(r.Context(), uploadDir, partName, false, false); err != nil {
|
||||
upload := s3a.uploadFiler(dstBucket, dstObject)
|
||||
if exists, _ := upload.exists(uploadDir, partName, false); exists {
|
||||
if err := upload.rm(r.Context(), uploadDir, partName, false, false); err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := s3a.mkFile(uploadDir, partName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
|
||||
if err := upload.mkFile(uploadDir, partName, dstEntry.Chunks, func(entry *filer_pb.Entry) {
|
||||
entry.Attributes = dstEntry.Attributes
|
||||
entry.Extended = dstEntry.Extended
|
||||
}); err != nil {
|
||||
|
||||
@@ -442,7 +442,7 @@ func TestWriteEmptyCopyPartStoresMd5(t *testing.T) {
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, filerImpl))
|
||||
util_http.InitGlobalHttpClient()
|
||||
|
||||
etag, code := s3a.writeEmptyCopyPart("b", "u1", 3)
|
||||
etag, code := s3a.writeEmptyCopyPart("b", "dst", "u1", 3)
|
||||
if code != s3err.ErrNone {
|
||||
t.Fatalf("writeEmptyCopyPart: %v", code)
|
||||
}
|
||||
|
||||
@@ -374,7 +374,7 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption(
|
||||
uploadEntry *filer_pb.Entry,
|
||||
) (etag string, sseMetadata SSEResponseMetadata, errCode s3err.ErrorCode) {
|
||||
if endOffset < startOffset && !uploadEntryHasChecksum(uploadEntry) {
|
||||
tag, code := s3a.writeEmptyCopyPart(dstBucket, uploadID, partID)
|
||||
tag, code := s3a.writeEmptyCopyPart(dstBucket, dstObject, uploadID, partID)
|
||||
return tag, SSEResponseMetadata{}, code
|
||||
}
|
||||
sliceLen := int64(0)
|
||||
@@ -404,7 +404,7 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption(
|
||||
applyDestChecksumHeaderToCopyRequest(cloned, uploadEntry)
|
||||
|
||||
filePath := s3a.genPartUploadPath(dstBucket, uploadID, partID)
|
||||
tag, code, putSSE := s3a.putToFiler(withMultipartPart(cloned), filePath, srcReader, dstBucket, "", partID, 0, nil, false, s3a.toFilerPath(dstBucket, dstObject))
|
||||
tag, code, putSSE := s3a.putToFiler(withMultipartPart(cloned, s3a.uploadFiler(dstBucket, dstObject)), filePath, srcReader, dstBucket, "", partID, 0, nil, false, s3a.toFilerPath(dstBucket, dstObject))
|
||||
if code != s3err.ErrNone {
|
||||
return "", SSEResponseMetadata{}, code
|
||||
}
|
||||
@@ -413,15 +413,16 @@ func (s3a *S3ApiServer) copyObjectPartViaReencryption(
|
||||
|
||||
// writeEmptyCopyPart writes a 0-byte part entry for an empty UploadPartCopy
|
||||
// range, mirroring the legacy fast path's handling of endOffset < startOffset.
|
||||
func (s3a *S3ApiServer) writeEmptyCopyPart(dstBucket, uploadID string, partID int) (string, s3err.ErrorCode) {
|
||||
func (s3a *S3ApiServer) writeEmptyCopyPart(dstBucket, dstObject, uploadID string, partID int) (string, s3err.ErrorCode) {
|
||||
uploadDir, partName := s3a.copyPartLocation(dstBucket, uploadID, partID)
|
||||
if exists, _ := s3a.exists(uploadDir, partName, false); exists {
|
||||
if err := s3a.rm(context.Background(), uploadDir, partName, false, false); err != nil {
|
||||
upload := s3a.uploadFiler(dstBucket, dstObject)
|
||||
if exists, _ := upload.exists(uploadDir, partName, false); exists {
|
||||
if err := upload.rm(context.Background(), uploadDir, partName, false, false); err != nil {
|
||||
return "", s3err.ErrInternalError
|
||||
}
|
||||
}
|
||||
const emptyMD5Hex = "d41d8cd98f00b204e9800998ecf8427e"
|
||||
if err := s3a.mkFile(uploadDir, partName, nil, func(e *filer_pb.Entry) {
|
||||
if err := upload.mkFile(uploadDir, partName, nil, func(e *filer_pb.Entry) {
|
||||
if e.Attributes == nil {
|
||||
e.Attributes = &filer_pb.FuseAttributes{}
|
||||
}
|
||||
|
||||
@@ -413,7 +413,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
|
||||
glog.V(2).Infof("PutObjectPartHandler %s %s %04d", bucket, uploadID, partID)
|
||||
|
||||
// Verify the multipart upload exists (rejects parts after abort)
|
||||
uploadEntry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
uploadEntry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchUpload)
|
||||
return
|
||||
@@ -514,7 +514,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
|
||||
// start the TTL clock before CompleteMultipartUpload ever assembled
|
||||
// the object. filer.conf storage rules are the opposite case: they
|
||||
// place the bytes the part becomes, so they resolve against the object.
|
||||
etag, errCode, sseMetadata := s3a.putToFiler(withMultipartPart(r), filePath, dataReader, bucket, "", partID, 0, nil, false, s3a.toFilerPath(bucket, object))
|
||||
etag, errCode, sseMetadata := s3a.putToFiler(withMultipartPart(r, s3a.uploadFiler(bucket, object)), filePath, dataReader, bucket, "", partID, 0, nil, false, s3a.toFilerPath(bucket, object))
|
||||
if errCode != s3err.ErrNone {
|
||||
glog.Errorf("PutObjectPart: putToFiler failed with error code %v for bucket=%s, object=%s, partNumber=%d",
|
||||
errCode, bucket, object, partID)
|
||||
@@ -554,7 +554,7 @@ func isMultipartUploadEntry(entry *filer_pb.Entry) bool {
|
||||
// part along with the directory it resurrected when an abort was answered while
|
||||
// the part was in flight. It reports whether the caller may answer success.
|
||||
func (s3a *S3ApiServer) checkUploadStillOpen(w http.ResponseWriter, r *http.Request, bucket, object, uploadID string) bool {
|
||||
entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
entry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
glog.Errorf("checkUploadStillOpen %s/%s: %v", bucket, uploadID, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
@@ -580,8 +580,8 @@ func (s3a *S3ApiServer) checkUploadStillOpen(w http.ResponseWriter, r *http.Requ
|
||||
// if the upload entry is not found or had no SSE. It is used by the bucket
|
||||
// policy engine to evaluate s3:x-amz-server-side-encryption conditions for
|
||||
// UploadPart and UploadPartCopy, which do not re-send the SSE header.
|
||||
func (s3a *S3ApiServer) getMultipartSSEAlgorithm(bucket, uploadID string) string {
|
||||
entry, err := s3a.getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
func (s3a *S3ApiServer) getMultipartSSEAlgorithm(bucket, object, uploadID string) string {
|
||||
entry, err := s3a.uploadFiler(bucket, object).getEntry(s3a.genUploadsFolder(bucket), uploadID)
|
||||
if err != nil || entry == nil || entry.Extended == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -540,13 +540,14 @@ func (s3a *S3ApiServer) uploadChunkSize() int32 {
|
||||
}
|
||||
|
||||
// multipartPartContextKey marks a putToFiler request as writing a staged
|
||||
// multipart part. Encryption for the upload is decided at
|
||||
// CreateMultipartUpload and replayed through request headers, so bucket
|
||||
// default encryption must not be re-evaluated for part writes.
|
||||
// multipart part, carrying the filer the upload's directory lives on.
|
||||
// Encryption for the upload is decided at CreateMultipartUpload and replayed
|
||||
// through request headers, so bucket default encryption must not be
|
||||
// re-evaluated for part writes.
|
||||
type multipartPartContextKey struct{}
|
||||
|
||||
func withMultipartPart(r *http.Request) *http.Request {
|
||||
return r.WithContext(context.WithValue(r.Context(), multipartPartContextKey{}, true))
|
||||
func withMultipartPart(r *http.Request, upload uploadFilerClient) *http.Request {
|
||||
return r.WithContext(context.WithValue(r.Context(), multipartPartContextKey{}, upload))
|
||||
}
|
||||
|
||||
// validatePutHeaders checks x-amz-storage-class and x-amz-tagging, so the
|
||||
@@ -572,10 +573,15 @@ func validatePutHeaders(r *http.Request) (map[string]string, s3err.ErrorCode) {
|
||||
}
|
||||
|
||||
func isMultipartPartRequest(r *http.Request) bool {
|
||||
marked, _ := r.Context().Value(multipartPartContextKey{}).(bool)
|
||||
_, marked := multipartPartUpload(r)
|
||||
return marked
|
||||
}
|
||||
|
||||
func multipartPartUpload(r *http.Request) (uploadFilerClient, bool) {
|
||||
upload, marked := r.Context().Value(multipartPartContextKey{}).(uploadFilerClient)
|
||||
return upload, marked
|
||||
}
|
||||
|
||||
// putToFiler writes one chunk of object bytes (a full PutObject body, a
|
||||
// single MPU part, a copy-part destination). lifecycleTTLSec is non-zero
|
||||
// only for top-level PutObject paths where the lifecycle XML's
|
||||
@@ -1076,6 +1082,8 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
case codes.Unavailable, codes.Canceled, codes.DeadlineExceeded:
|
||||
s3a.markOwnerUnreachable(owner)
|
||||
}
|
||||
} else if upload, ok := multipartPartUpload(r); ok && cond == nil {
|
||||
createErr = upload.WithFilerClient(false, create)
|
||||
} else if s3a.filerClient != nil {
|
||||
createErr = s3a.withFilerClientFailover(context.Background(), "", false, cond == nil, create)
|
||||
} else {
|
||||
@@ -1184,7 +1192,11 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader
|
||||
// can hide a commit (issue #11366) and the filer can fail after
|
||||
// inserting the entry (issue #11387), so the entry's presence — not
|
||||
// the error class — decides the chunks' fate.
|
||||
landed, absent := s3a.confirmCreateLanded(filePath, bucket, object, entry, chunkResult.FileChunks, finalize)
|
||||
routeObject := object
|
||||
if upload, ok := multipartPartUpload(r); ok {
|
||||
routeObject = upload.object
|
||||
}
|
||||
landed, absent := s3a.confirmCreateLanded(filePath, bucket, object, routeObject, entry, chunkResult.FileChunks, finalize)
|
||||
if landed {
|
||||
createCode = s3err.ErrNone
|
||||
}
|
||||
@@ -1397,16 +1409,17 @@ func chunkNeedleExists(ctx context.Context, fileUrl, jwt string) bool {
|
||||
// finalization the error skipped then runs under the object write lock, and
|
||||
// landed reports success — while absent requires every filer the create could
|
||||
// have committed on to lack the entry, the only outcome where the uploaded
|
||||
// chunks are orphaned.
|
||||
func (s3a *S3ApiServer) confirmCreateLanded(filePath, bucket, object string, entry *filer_pb.Entry, uploaded []*filer_pb.FileChunk, finalize *putFinalize) (landed, absent bool) {
|
||||
// chunks are orphaned. routeObject picks the owner filers to check: the object
|
||||
// itself, or for a staged part the object its upload completes into.
|
||||
func (s3a *S3ApiServer) confirmCreateLanded(filePath, bucket, object, routeObject string, entry *filer_pb.Entry, uploaded []*filer_pb.FileChunk, finalize *putFinalize) (landed, absent bool) {
|
||||
dir, name := path.Dir(filePath), path.Base(filePath)
|
||||
owner := s3a.routableWriteOwner(bucket, object)
|
||||
owner := s3a.routableWriteOwner(bucket, routeObject)
|
||||
lookupCtx, cancel := context.WithTimeout(context.Background(), createLookupTimeout)
|
||||
defer cancel()
|
||||
// Verify, finalize, and roll back inside one critical section: a concurrent
|
||||
// write to the same key must not slip in between them.
|
||||
s3a.withObjectWriteLock(bucket, object, nil, func() s3err.ErrorCode {
|
||||
targets := s3a.createTargetFilers(owner, bucket, object)
|
||||
targets := s3a.createTargetFilers(owner, bucket, routeObject)
|
||||
existing, uncertain := s3a.lookupCommittedEntry(lookupCtx, targets, dir, name)
|
||||
if existing == nil {
|
||||
absent = len(targets) > 0 && !uncertain
|
||||
@@ -2594,11 +2607,8 @@ func (s3a *S3ApiServer) validateObjectLockHeaders(r *http.Request, bucket, objec
|
||||
}
|
||||
}
|
||||
|
||||
// Check for governance bypass header - only valid for buckets with Object Lock enabled
|
||||
bypassGovernance := r.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
|
||||
// Governance bypass headers are only valid for buckets with Object Lock enabled
|
||||
if bypassGovernance && !objectLockEnabled {
|
||||
if governanceBypassRequested(r) && !objectLockEnabled {
|
||||
return ErrGovernanceBypassVersioningRequired
|
||||
}
|
||||
|
||||
|
||||
@@ -155,7 +155,7 @@ type ambiguousPutFiler struct {
|
||||
// beforeLookup runs inside LookupDirectoryEntry, letting a test delete a
|
||||
// just-committed entry before recovery reads it.
|
||||
beforeCreate func()
|
||||
beforeLookup func()
|
||||
beforeLookup func(key string)
|
||||
beforeAssign func()
|
||||
stores int
|
||||
conds []*filer_pb.WriteCondition
|
||||
@@ -232,11 +232,18 @@ func putConditionSatisfied(cond *filer_pb.WriteCondition, current *filer_pb.Entr
|
||||
return true
|
||||
}
|
||||
|
||||
func (f *ambiguousPutFiler) DeleteEntry(_ context.Context, req *filer_pb.DeleteEntryRequest) (*filer_pb.DeleteEntryResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
delete(f.entries, req.Directory+"/"+req.Name)
|
||||
return &filer_pb.DeleteEntryResponse{}, nil
|
||||
}
|
||||
|
||||
func (f *ambiguousPutFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.beforeLookup != nil {
|
||||
f.beforeLookup()
|
||||
f.beforeLookup(req.Directory + "/" + req.Name)
|
||||
}
|
||||
if f.lookupErr != nil && req.Directory+"/"+req.Name == f.lookupFailKey {
|
||||
return nil, f.lookupErr
|
||||
@@ -279,28 +286,23 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, clause := range req.GetCondition().GetClauses() {
|
||||
var md5Hex string
|
||||
if current != nil && current.Attributes != nil {
|
||||
md5Hex = fmt.Sprintf("%x", current.Attributes.Md5)
|
||||
}
|
||||
ok := true
|
||||
switch clause.Kind {
|
||||
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
||||
ok = current == nil
|
||||
case filer_pb.WriteCondition_IF_EXISTS:
|
||||
ok = current != nil
|
||||
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
||||
ok = current != nil && etagCandidateMatch(clause.Etags, md5Hex)
|
||||
case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH:
|
||||
ok = current == nil || !etagCandidateMatch(clause.Etags, md5Hex)
|
||||
}
|
||||
if !ok {
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
|
||||
}
|
||||
if !txnConditionHolds(req.GetCondition(), current) {
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED}, nil
|
||||
}
|
||||
if !f.txnNoApply {
|
||||
for _, m := range req.Mutations {
|
||||
for i, m := range req.Mutations {
|
||||
if m.Condition != nil {
|
||||
target := m.ConditionKey
|
||||
if target == "" && m.Type == filer_pb.ObjectMutation_PUT {
|
||||
target = m.Directory + "/" + m.Entry.Name
|
||||
} else if target == "" {
|
||||
target = m.Directory + "/" + m.Name
|
||||
}
|
||||
if !txnConditionHolds(m.Condition, f.entries[target]) {
|
||||
failed := int32(i)
|
||||
return &filer_pb.ObjectTransactionResponse{ErrorCode: filer_pb.FilerError_PRECONDITION_FAILED, FailedMutation: &failed}, nil
|
||||
}
|
||||
}
|
||||
switch m.Type {
|
||||
case filer_pb.ObjectMutation_PUT:
|
||||
entry := proto.Clone(m.Entry).(*filer_pb.Entry)
|
||||
@@ -321,6 +323,36 @@ func (f *ambiguousPutFiler) ObjectTransaction(_ context.Context, req *filer_pb.O
|
||||
return &filer_pb.ObjectTransactionResponse{}, f.txnErr
|
||||
}
|
||||
|
||||
// txnConditionHolds evaluates an ObjectTransaction condition, request-level or
|
||||
// on one mutation, the way the filer does for the kinds the tests send.
|
||||
func txnConditionHolds(cond *filer_pb.WriteCondition, current *filer_pb.Entry) bool {
|
||||
for _, clause := range cond.GetClauses() {
|
||||
var etag string
|
||||
if v := current.GetExtended()[s3_constants.ExtETagKey]; len(v) > 0 {
|
||||
etag = strings.Trim(string(v), `"`)
|
||||
} else if current != nil && current.Attributes != nil {
|
||||
etag = fmt.Sprintf("%x", current.Attributes.Md5)
|
||||
}
|
||||
ok := true
|
||||
switch clause.Kind {
|
||||
case filer_pb.WriteCondition_IF_NOT_EXISTS:
|
||||
ok = current == nil
|
||||
case filer_pb.WriteCondition_IF_EXISTS:
|
||||
ok = current != nil
|
||||
case filer_pb.WriteCondition_IF_ETAG_MATCH:
|
||||
ok = current != nil && etagCandidateMatch(clause.Etags, etag)
|
||||
case filer_pb.WriteCondition_IF_ETAG_NOT_MATCH:
|
||||
ok = current == nil || !etagCandidateMatch(clause.Etags, etag)
|
||||
case filer_pb.WriteCondition_IF_ENTRY_EQUAL:
|
||||
ok = (current == nil) == (clause.ExpectedEntry == nil) && (current == nil || proto.Equal(current, clause.ExpectedEntry))
|
||||
}
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// etagCandidateMatch mirrors the filer's strong-comparison canonicalization:
|
||||
// quotes are stripped on both sides and a weak (W/) candidate never matches.
|
||||
func etagCandidateMatch(candidates []string, stored string) bool {
|
||||
@@ -953,7 +985,7 @@ func TestPutToFilerRoutedAmbiguousEmptyPutAbsentNotRecreated(t *testing.T) {
|
||||
txnErr: status.Error(codes.Unavailable, "response lost after commit"),
|
||||
}
|
||||
// The committed empty object is deleted before recovery looks it up.
|
||||
filerImpl.beforeLookup = func() {
|
||||
filerImpl.beforeLookup = func(string) {
|
||||
delete(filerImpl.entries, "/buckets/b/o")
|
||||
}
|
||||
filerAddr := startFakeFiler(t, filerImpl)
|
||||
@@ -1277,3 +1309,12 @@ func TestSuspendedPutIfMatchRecheckedWhenAltKeyIgnored(t *testing.T) {
|
||||
t.Fatalf("condition keys = %v, want [\"\" \"/buckets/b/k.versions/v_v1\"]", conditionKeys)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTxnConditionHoldsQuotedStoredETag(t *testing.T) {
|
||||
stored := &filer_pb.Entry{Extended: map[string][]byte{s3_constants.ExtETagKey: []byte(`"abc123"`)}}
|
||||
match := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_MATCH, Etags: []string{`"abc123"`}}}}
|
||||
notMatch := &filer_pb.WriteCondition{Clauses: []*filer_pb.WriteCondition_Clause{{Kind: filer_pb.WriteCondition_IF_ETAG_NOT_MATCH, Etags: []string{"abc123"}}}}
|
||||
if !txnConditionHolds(match, stored) || txnConditionHolds(notMatch, stored) {
|
||||
t.Fatalf("a quoted stored ETag must compare equal to the same ETag")
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,38 @@ func TestGovernanceBypassDoesNotInheritDeleteObjectVersion(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateGovernanceBypassRequestHeaderCase(t *testing.T) {
|
||||
iam := &IdentityAccessManagement{}
|
||||
require.NoError(t, iam.PutPolicy("BypassGovernance",
|
||||
`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:DeleteObjectVersion","s3:BypassGovernanceRetention"],"Resource":"arn:aws:s3:::worm/*"}]}`))
|
||||
s3a := &S3ApiServer{iam: iam}
|
||||
|
||||
for _, tc := range []struct {
|
||||
header string
|
||||
want bool
|
||||
}{
|
||||
{"true", true},
|
||||
{"True", true},
|
||||
{"TRUE", true},
|
||||
{"false", false},
|
||||
{"", false},
|
||||
} {
|
||||
req := httptest.NewRequest(http.MethodDelete, "http://localhost:8333/worm/doc.txt?versionId=abc123", nil)
|
||||
req = mux.SetURLVars(req, map[string]string{"bucket": "worm", "object": "doc.txt"})
|
||||
if tc.header != "" {
|
||||
req.Header.Set(s3_constants.AmzBypassGovernanceRetention, tc.header)
|
||||
}
|
||||
req = req.WithContext(s3_constants.SetIdentityInContext(req.Context(), &Identity{
|
||||
Name: "breaker",
|
||||
PolicyNames: []string{"BypassGovernance"},
|
||||
Account: &Account{Id: "test-account"},
|
||||
}))
|
||||
if got := s3a.evaluateGovernanceBypassRequest(req, "worm", "/doc.txt"); got != tc.want {
|
||||
t.Errorf("header %q: evaluateGovernanceBypassRequest() = %v, want %v", tc.header, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The object-lock request headers on PutObject/CreateMultipartUpload must be
|
||||
// authorized as s3:PutObjectRetention / s3:PutObjectLegalHold; s3:PutObject
|
||||
// alone used to set retention and legal hold on new versions.
|
||||
|
||||
@@ -520,6 +520,14 @@ func TestValidateObjectLockHeaders(t *testing.T) {
|
||||
assert.True(t, errors.Is(err, ErrGovernanceBypassVersioningRequired))
|
||||
})
|
||||
|
||||
t.Run("Title-case governance bypass header on non-versioned bucket", func(t *testing.T) {
|
||||
req := httptest.NewRequest("PUT", "/bucket/object", nil)
|
||||
req.Header.Set("x-amz-bypass-governance-retention", "True")
|
||||
|
||||
err := s3a.validateObjectLockHeaders(req, "bucket", "object", false)
|
||||
assert.True(t, errors.Is(err, ErrGovernanceBypassVersioningRequired))
|
||||
})
|
||||
|
||||
t.Run("Governance bypass header on versioned bucket should pass", func(t *testing.T) {
|
||||
req := httptest.NewRequest("PUT", "/bucket/object", nil)
|
||||
req.Header.Set("x-amz-bypass-governance-retention", "true")
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -543,11 +544,15 @@ func (s3a *S3ApiServer) checkGovernanceBypassPermission(request *http.Request, b
|
||||
return true
|
||||
}
|
||||
|
||||
// AWS SDKs send the boolean as "true" or "True".
|
||||
func governanceBypassRequested(r *http.Request) bool {
|
||||
return strings.EqualFold(r.Header.Get(s3_constants.AmzBypassGovernanceRetention), "true")
|
||||
}
|
||||
|
||||
// evaluateGovernanceBypassRequest evaluates if governance bypass is requested and permitted
|
||||
func (s3a *S3ApiServer) evaluateGovernanceBypassRequest(r *http.Request, bucket, object string) bool {
|
||||
// Step 1: Check if governance bypass was requested via header
|
||||
bypassRequested := r.Header.Get("x-amz-bypass-governance-retention") == "true"
|
||||
if !bypassRequested {
|
||||
if !governanceBypassRequested(r) {
|
||||
// No bypass requested - normal retention enforcement applies
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sort"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
// uploadFilerClient reads and writes one multipart upload's .uploads/<id>
|
||||
// directory on the destination object's write owner, the filer the upload's
|
||||
// completion commits on. Gateways on independent filer stores then see an
|
||||
// upload, and its parts, as soon as the owner stored them.
|
||||
type uploadFilerClient struct {
|
||||
s3a *S3ApiServer
|
||||
bucket string
|
||||
object string
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) uploadFiler(bucket, object string) uploadFilerClient {
|
||||
return uploadFilerClient{s3a: s3a, bucket: bucket, object: object}
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error {
|
||||
owner := u.s3a.routableWriteOwner(u.bucket, u.object)
|
||||
if owner == "" || u.s3a.filerClient == nil {
|
||||
return u.s3a.WithFilerClient(streamingMode, fn)
|
||||
}
|
||||
if u.s3a.ownerRecentlyUnreachable(owner) {
|
||||
owner = ""
|
||||
}
|
||||
return u.s3a.withFilerClientFailover(context.Background(), owner, streamingMode, true, fn)
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) AdjustedUrl(location *filer_pb.Location) string {
|
||||
return u.s3a.AdjustedUrl(location)
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) GetDataCenter() string {
|
||||
return u.s3a.GetDataCenter()
|
||||
}
|
||||
|
||||
// getEntry reads the upload record. Right after a ring change a part write can
|
||||
// leave the new owner a bare directory before the record replicated there, so
|
||||
// the prior owner's record wins over one without the upload's key, and a prior
|
||||
// owner that cannot answer fails the lookup rather than reading as no upload.
|
||||
func (u uploadFilerClient) getEntry(parentDirectoryPath, entryName string) (*filer_pb.Entry, error) {
|
||||
entry, err := u.s3a.getObjectScopedEntryRoutedByKey(u.bucket, u.object, parentDirectoryPath, entryName)
|
||||
if (err != nil && !errors.Is(err, filer_pb.ErrNotFound)) || isMultipartUploadEntry(entry) {
|
||||
return entry, err
|
||||
}
|
||||
prior := u.s3a.priorWriteOwner(u.bucket, u.object)
|
||||
if prior == "" {
|
||||
return entry, err
|
||||
}
|
||||
priorEntry, priorErr := u.s3a.lookupEntryOnFiler(context.Background(), prior, parentDirectoryPath, entryName)
|
||||
if priorErr != nil && !isFilerNotFound(priorErr) {
|
||||
return nil, priorErr
|
||||
}
|
||||
if priorErr == nil && isMultipartUploadEntry(priorEntry) {
|
||||
return priorEntry, nil
|
||||
}
|
||||
return entry, err
|
||||
}
|
||||
|
||||
// list reads the owner's listing. While a ring change is in its cooling-off
|
||||
// window the prior owner may still hold parts the new owner has not received,
|
||||
// so its listing is merged in, the owner's entry winning a name both hold.
|
||||
func (u uploadFilerClient) list(parentDirectoryPath, prefix, startFrom string, inclusive bool, limit uint32) ([]*filer_pb.Entry, bool, error) {
|
||||
entries, isLast, err := u.s3a.listObjectScopedEntriesRoutedByKey(u.bucket, u.object, parentDirectoryPath, prefix, startFrom, inclusive, limit)
|
||||
if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return entries, isLast, err
|
||||
}
|
||||
prior := u.s3a.priorWriteOwner(u.bucket, u.object)
|
||||
if prior == "" {
|
||||
return entries, isLast, err
|
||||
}
|
||||
var priorEntries []*filer_pb.Entry
|
||||
priorLast := false
|
||||
priorErr := pb.WithFilerClient(true, 0, prior, u.s3a.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return filer_pb.SeaweedList(context.Background(), client, parentDirectoryPath, prefix, func(entry *filer_pb.Entry, isLastEntry bool) error {
|
||||
priorEntries = append(priorEntries, entry)
|
||||
priorLast = priorLast || isLastEntry
|
||||
return nil
|
||||
}, startFrom, inclusive, limit)
|
||||
})
|
||||
if priorErr != nil && !isFilerNotFound(priorErr) && len(entries) == 0 {
|
||||
// with nothing on the owner yet, every part may be on the prior owner
|
||||
return nil, false, priorErr
|
||||
}
|
||||
if priorErr != nil || len(priorEntries) == 0 {
|
||||
return entries, isLast, err
|
||||
}
|
||||
merged, mergedLast := mergeEntryListings(entries, isLast, priorEntries, priorLast, limit)
|
||||
return merged, mergedLast, nil
|
||||
}
|
||||
|
||||
// mergeEntryListings merges two pages of one directory listed from the same
|
||||
// cursor, taking primary's entry for a name both hold. Each page holds the
|
||||
// first limit names past the cursor, so the merged first limit are exact.
|
||||
func mergeEntryListings(primary []*filer_pb.Entry, primaryLast bool, secondary []*filer_pb.Entry, secondaryLast bool, limit uint32) ([]*filer_pb.Entry, bool) {
|
||||
byName := make(map[string]*filer_pb.Entry, len(primary)+len(secondary))
|
||||
for _, entry := range secondary {
|
||||
byName[entry.Name] = entry
|
||||
}
|
||||
for _, entry := range primary {
|
||||
byName[entry.Name] = entry
|
||||
}
|
||||
merged := make([]*filer_pb.Entry, 0, len(byName))
|
||||
for _, entry := range byName {
|
||||
merged = append(merged, entry)
|
||||
}
|
||||
sort.Slice(merged, func(i, j int) bool { return merged[i].Name < merged[j].Name })
|
||||
isLast := primaryLast && secondaryLast
|
||||
if limit > 0 && uint32(len(merged)) > limit {
|
||||
merged, isLast = merged[:limit], false
|
||||
}
|
||||
return merged, isLast
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) exists(parentDirectoryPath, entryName string, isDirectory bool) (bool, error) {
|
||||
return filer_pb.Exists(context.Background(), u, parentDirectoryPath, entryName, isDirectory)
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) mkdir(parentDirectoryPath, dirName string, fn func(entry *filer_pb.Entry)) error {
|
||||
return filer_pb.Mkdir(context.Background(), u, parentDirectoryPath, dirName, fn)
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) mkFile(parentDirectoryPath, fileName string, chunks []*filer_pb.FileChunk, fn func(entry *filer_pb.Entry)) error {
|
||||
return filer_pb.MkFile(context.Background(), u, parentDirectoryPath, fileName, chunks, fn)
|
||||
}
|
||||
|
||||
func (u uploadFilerClient) rm(ctx context.Context, parentDirectoryPath, entryName string, isDeleteData, isRecursive bool) error {
|
||||
return retryFilerOp(ctx, "rm "+parentDirectoryPath+"/"+entryName, func() error {
|
||||
return u.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
return doDeleteEntry(ctx, client, parentDirectoryPath, entryName, isDeleteData, isRecursive)
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,350 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go/aws"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// The gateway's own filer never receives the owner's metadata here, so every
|
||||
// multipart step after CreateMultipartUpload must find the upload and its
|
||||
// parts on the object's write owner.
|
||||
func TestMultipartUploadOnOwnerFiler(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
replica := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
ownerAddr := startFakeFiler(t, owner)
|
||||
replicaAddr := startFakeFiler(t, replica)
|
||||
s3a := newPutTestServer(t, replicaAddr)
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1)
|
||||
|
||||
has := func(f *ambiguousPutFiler, path string) bool {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.entries[path] != nil
|
||||
}
|
||||
partsUnder := func(f *ambiguousPutFiler, dir string) int {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
n := 0
|
||||
for key, entry := range f.entries {
|
||||
if strings.HasPrefix(key, dir+"/") && !entry.IsDirectory {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
initiate := func() string {
|
||||
r := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil)
|
||||
out, code := s3a.createMultipartUpload(r, &s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
require.True(t, has(owner, uploadDir), "the upload must be created on the owner")
|
||||
require.False(t, has(replica, uploadDir), "the gateway's filer must still be behind")
|
||||
return *out.UploadId
|
||||
}
|
||||
uploadPart := func(uploadID string, partNumber int, body string) string {
|
||||
r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber, uploadID), strings.NewReader(body))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
etag := w.Header()["ETag"]
|
||||
require.Len(t, etag, 1)
|
||||
return etag[0]
|
||||
}
|
||||
|
||||
uploadID := initiate()
|
||||
uploadDir := "/buckets/b/.uploads/" + uploadID
|
||||
etag1 := uploadPart(uploadID, 1, strings.Repeat("a", 5*1024*1024))
|
||||
etag2 := uploadPart(uploadID, 2, "tail")
|
||||
require.Equal(t, 2, partsUnder(owner, uploadDir), "parts must be stored next to the upload")
|
||||
require.Equal(t, 0, partsUnder(replica, uploadDir))
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID),
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 2)
|
||||
|
||||
complete := httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+uploadID, nil)
|
||||
_, code = s3a.completeMultipartUpload(complete, &s3.CompleteMultipartUploadInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(uploadID),
|
||||
}, &CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etag1, PartNumber: 1}, {ETag: etag2, PartNumber: 2}}})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.True(t, has(owner, "/buckets/b/dir/object"), "the object must commit on the owner")
|
||||
require.False(t, has(owner, uploadDir), "completion must remove the upload")
|
||||
|
||||
aborted := initiate()
|
||||
uploadPart(aborted, 1, "data")
|
||||
_, code = s3a.abortMultipartUpload(&s3.AbortMultipartUploadInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: aws.String(aborted),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.False(t, has(owner, "/buckets/b/.uploads/"+aborted), "abort must remove the upload")
|
||||
}
|
||||
|
||||
// A part create the owner stored but answered with an error must be found
|
||||
// there before its chunks are freed, even though the owner is not one of the
|
||||
// gateway's own filers.
|
||||
func TestMultipartPartCreateErrorChecksUploadOwner(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
owner := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{},
|
||||
respError: "create parent directories: i/o timeout"}
|
||||
replica := &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}
|
||||
ownerAddr := startFakeFiler(t, owner)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, replica))
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, ownerAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{ownerAddr}, 1)
|
||||
|
||||
r := withMultipartPart(httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId=up1", nil), s3a.uploadFiler("b", "dir/object"))
|
||||
etag, code, _ := s3a.putToFiler(r, "/buckets/b/.uploads/up1/0001_x.part", strings.NewReader("part data"), "b", "", 1, 0, nil, false, "/buckets/b/dir/object")
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.NotEmpty(t, etag)
|
||||
require.Empty(t, volume.deleted(), "chunks under the stored part must survive")
|
||||
}
|
||||
|
||||
// Right after the key moves to a new owner that has not received the upload
|
||||
// yet, ListParts finds the record and its parts on the prior owner.
|
||||
func TestListPartsAfterOwnerChange(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
require.Equal(t, priorAddr, s3a.priorWriteOwner("b", "dir/object"))
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 1)
|
||||
}
|
||||
|
||||
// A part routed to a new owner that has not received the upload yet makes that
|
||||
// filer create a bare .uploads/<id> directory; the upload must stay open.
|
||||
func TestUploadPartAfterOwnerChange(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
next.beforeCreate = func() {
|
||||
if next.entries[uploadDir] == nil {
|
||||
next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true}
|
||||
}
|
||||
}
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
require.Len(t, listed.Part, 1)
|
||||
}
|
||||
|
||||
// A new owner that received only some of the parts so far lists them together
|
||||
// with the rest still on the prior owner.
|
||||
func TestListPartsMergesPriorOwnerParts(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
etags := map[int]string{}
|
||||
for partNumber, body := range []string{strings.Repeat("a", 5*1024*1024), "tail"} {
|
||||
r := httptest.NewRequest(http.MethodPut, fmt.Sprintf("/b/dir/object?partNumber=%d&uploadId=%s", partNumber+1, *out.UploadId), strings.NewReader(body))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
require.Len(t, w.Header()["ETag"], 1)
|
||||
etags[partNumber+1] = w.Header()["ETag"][0]
|
||||
}
|
||||
|
||||
prior.mu.Lock()
|
||||
for key, entry := range prior.entries {
|
||||
if key == uploadDir || strings.HasPrefix(key, uploadDir+"/0001") {
|
||||
next.entries[key] = entry
|
||||
}
|
||||
}
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
listParts := func(maxParts, marker int64) *ListPartsResult {
|
||||
listed, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(maxParts), PartNumberMarker: aws.Int64(marker),
|
||||
})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
return listed
|
||||
}
|
||||
require.Len(t, listParts(1000, 0).Part, 2)
|
||||
first := listParts(1, 0)
|
||||
require.Len(t, first.Part, 1)
|
||||
require.Equal(t, int64(1), *first.Part[0].PartNumber)
|
||||
require.True(t, *first.IsTruncated)
|
||||
second := listParts(1, 1)
|
||||
require.Len(t, second.Part, 1)
|
||||
require.Equal(t, int64(2), *second.Part[0].PartNumber)
|
||||
require.False(t, *second.IsTruncated)
|
||||
|
||||
_, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil),
|
||||
&s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId},
|
||||
&CompleteMultipartUpload{Parts: []CompletedPart{{ETag: etags[1], PartNumber: 1}, {ETag: etags[2], PartNumber: 2}}})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
}
|
||||
|
||||
type failingListFiler struct {
|
||||
*ambiguousPutFiler
|
||||
}
|
||||
|
||||
func (f failingListFiler) ListEntries(*filer_pb.ListEntriesRequest, filer_pb.SeaweedFiler_ListEntriesServer) error {
|
||||
return status.Error(codes.Unavailable, "store unavailable")
|
||||
}
|
||||
|
||||
// With no parts on the new owner yet, a prior owner that cannot be listed may
|
||||
// hold all of them, so completion must not read the upload as empty.
|
||||
func TestCompleteFailsWhenPriorOwnerListingFails(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, failingListFiler{prior})
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.iam = &IdentityAccessManagement{}
|
||||
s3a.bucketConfigCache = NewBucketConfigCache(time.Minute)
|
||||
s3a.bucketConfigCache.Set("b", &BucketConfig{Name: "b"})
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
r := httptest.NewRequest(http.MethodPut, "/b/dir/object?partNumber=1&uploadId="+*out.UploadId, strings.NewReader("data"))
|
||||
r = mux.SetURLVars(r, map[string]string{"bucket": "b", "object": "dir/object"})
|
||||
w := httptest.NewRecorder()
|
||||
s3a.PutObjectPartHandler(w, r)
|
||||
require.Equal(t, http.StatusOK, w.Code, w.Body.String())
|
||||
require.Len(t, w.Header()["ETag"], 1)
|
||||
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
prior.mu.Lock()
|
||||
next.entries[uploadDir] = prior.entries[uploadDir]
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
_, code = s3a.completeMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploadId="+*out.UploadId, nil),
|
||||
&s3.CompleteMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId},
|
||||
&CompleteMultipartUpload{Parts: []CompletedPart{{ETag: w.Header()["ETag"][0], PartNumber: 1}}})
|
||||
require.Equal(t, s3err.ErrInternalError, code)
|
||||
}
|
||||
|
||||
// A prior owner that cannot be asked for the upload record must not make an
|
||||
// upload the new owner has not received yet read as gone.
|
||||
func TestUploadLookupFailsWhenPriorOwnerUnreachable(t *testing.T) {
|
||||
util_http.InitGlobalHttpClient()
|
||||
volume := startFakeVolumeServer(t)
|
||||
prior := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
next := &ambiguousPutFiler{volume: volume, apply: true, entries: map[string]*filer_pb.Entry{}}
|
||||
priorAddr := startFakeFiler(t, prior)
|
||||
nextAddr := startFakeFiler(t, next)
|
||||
s3a := newPutTestServer(t, startFakeFiler(t, &ambiguousPutFiler{volume: volume, entries: map[string]*filer_pb.Entry{}}))
|
||||
s3a.objectWriteLockClient = cluster.NewLockClient(s3a.option.GrpcDialOption, priorAddr)
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{priorAddr}, 1)
|
||||
|
||||
out, code := s3a.createMultipartUpload(httptest.NewRequest(http.MethodPost, "/b/dir/object?uploads", nil),
|
||||
&s3.CreateMultipartUploadInput{Bucket: aws.String("b"), Key: aws.String("dir/object")})
|
||||
require.Equal(t, s3err.ErrNone, code)
|
||||
uploadDir := "/buckets/b/.uploads/" + *out.UploadId
|
||||
prior.mu.Lock()
|
||||
prior.lookupErr, prior.lookupFailKey = status.Error(codes.Unavailable, "store unavailable"), uploadDir
|
||||
prior.mu.Unlock()
|
||||
s3a.objectWriteLockClient.SetRing([]pb.ServerAddress{nextAddr}, 2)
|
||||
|
||||
listParts := func() s3err.ErrorCode {
|
||||
_, code := s3a.listObjectParts(&s3.ListPartsInput{
|
||||
Bucket: aws.String("b"), Key: aws.String("dir/object"), UploadId: out.UploadId,
|
||||
MaxParts: aws.Int64(1000), PartNumberMarker: aws.Int64(0),
|
||||
})
|
||||
return code
|
||||
}
|
||||
require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has nothing yet")
|
||||
next.mu.Lock()
|
||||
next.entries[uploadDir] = &filer_pb.Entry{Name: *out.UploadId, IsDirectory: true}
|
||||
next.mu.Unlock()
|
||||
require.Equal(t, s3err.ErrInternalError, listParts(), "new owner has only a bare directory")
|
||||
}
|
||||
@@ -323,17 +323,17 @@ type uploadRemovalTxn struct {
|
||||
// upload's directory inside the object's commit, after freeing the part
|
||||
// entries the object does not reference. It is nil when the write is not
|
||||
// routed and the upload directory still needs post-commit cleanup.
|
||||
func (s3a *S3ApiServer) routedUploadRemoval(ctx context.Context, owner pb.ServerAddress, uploadDirectory, bucket, uploadId string, completionState *multipartCompletionState) (*uploadRemovalTxn, error) {
|
||||
func (s3a *S3ApiServer) routedUploadRemoval(ctx context.Context, owner pb.ServerAddress, uploadDirectory, bucket, object, uploadId string, completionState *multipartCompletionState) (*uploadRemovalTxn, error) {
|
||||
if owner == "" {
|
||||
return nil, nil
|
||||
}
|
||||
// Older filers ignore the mutation's upload-exists condition, so during an
|
||||
// upgrade this gateway-side check keeps the same guard window the
|
||||
// pre-transaction code had.
|
||||
if exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadId, true); err == nil && !exists {
|
||||
if _, err := s3a.lookupEntryPreferringOwner(owner, s3a.genUploadsFolder(bucket), uploadId); errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return nil, errUploadRemoved
|
||||
}
|
||||
if err := s3a.deleteUnusedPartEntries(ctx, uploadDirectory, bucket, uploadId, completionState); err != nil {
|
||||
if err := s3a.deleteUnusedPartEntries(ctx, uploadDirectory, bucket, object, uploadId, completionState); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
conditionKey, condition := uploadExistsCondition(uploadDirectory)
|
||||
|
||||
@@ -649,10 +649,10 @@ func TestWriteMultipartObjectRemovesUploadDir(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if removal, err := s3a.routedUploadRemoval(context.Background(), "", "/buckets/b/.uploads/up1", "b", "up1", &multipartCompletionState{}); removal != nil || err != nil {
|
||||
if removal, err := s3a.routedUploadRemoval(context.Background(), "", "/buckets/b/.uploads/up1", "b", "o", "up1", &multipartCompletionState{}); removal != nil || err != nil {
|
||||
t.Fatalf("unrouted write should not carry the removal, got %+v, %v", removal, err)
|
||||
}
|
||||
removal, err := s3a.routedUploadRemoval(context.Background(), owner, "/buckets/b/.uploads/up1", "b", "up1", &multipartCompletionState{})
|
||||
removal, err := s3a.routedUploadRemoval(context.Background(), owner, "/buckets/b/.uploads/up1", "b", "o", "up1", &multipartCompletionState{})
|
||||
if err != nil {
|
||||
t.Fatalf("routedUploadRemoval: %v", err)
|
||||
}
|
||||
|
||||
@@ -74,8 +74,9 @@ func (s3a *S3ApiServer) latestPointerRecompute(bucket, object string, useInverte
|
||||
// owner filer, via a single RECOMPUTE_LATEST. versionEntry, when set, is PUT
|
||||
// into .versions in the same transaction, so the recompute's scan sees it even
|
||||
// when the gateway's own filer is not the owner; otherwise the version file is
|
||||
// already written on the owner.
|
||||
func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry) s3err.ErrorCode {
|
||||
// already written on the owner. latestGuard, when set, must hold on the
|
||||
// .versions directory entry under the same lock.
|
||||
func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket, object string, useInvertedFormat bool, versionEntry *filer_pb.Entry, latestGuard *filer_pb.WriteCondition) s3err.ErrorCode {
|
||||
mutations := []*filer_pb.ObjectMutation{s3a.latestPointerRecompute(bucket, object, useInvertedFormat, "", true)}
|
||||
if versionEntry != nil {
|
||||
mutations = append([]*filer_pb.ObjectMutation{{
|
||||
@@ -89,11 +90,17 @@ func (s3a *S3ApiServer) routedVersionedFinalize(owner pb.ServerAddress, bucket,
|
||||
RouteKey: s3a.objectRouteKey(bucket, object),
|
||||
Mutations: mutations,
|
||||
}
|
||||
if latestGuard != nil {
|
||||
req.Condition = latestGuard
|
||||
req.ConditionKey = s3a.toFilerPath(bucket, object+s3_constants.VersionsFolder)
|
||||
}
|
||||
resp, err := s3a.objectTxnOnFiler(owner, req)
|
||||
switch {
|
||||
case err != nil:
|
||||
glog.Errorf("routedVersionedFinalize: %s/%s on %s: %v", bucket, object, owner, err)
|
||||
return s3err.ErrInternalError
|
||||
case resp.ErrorCode == filer_pb.FilerError_PRECONDITION_FAILED:
|
||||
return s3err.ErrPreconditionFailed
|
||||
case resp.Error != "":
|
||||
glog.Errorf("routedVersionedFinalize: %s/%s: %s", bucket, object, resp.Error)
|
||||
return s3err.ErrInternalError
|
||||
@@ -158,7 +165,7 @@ func (s3a *S3ApiServer) routedMultipartFinalize(owner pb.ServerAddress, bucket,
|
||||
// A filer older than the mutation-condition support silently ignores
|
||||
// putCondition, so on upgrade the guard degrades to this gateway-side
|
||||
// check — the same window the pre-transaction code had.
|
||||
if exists, err := s3a.exists(s3a.genUploadsFolder(bucket), uploadID, true); err == nil && !exists {
|
||||
if exists, err := s3a.uploadFiler(bucket, object).exists(s3a.genUploadsFolder(bucket), uploadID, true); err == nil && !exists {
|
||||
return s3err.ErrNoSuchUpload
|
||||
}
|
||||
putConditionKey, putCondition := uploadExistsCondition(s3a.genUploadsFolder(bucket) + "/" + uploadID)
|
||||
|
||||
@@ -201,8 +201,18 @@ type ObjectVersion struct {
|
||||
StorageClass string
|
||||
}
|
||||
|
||||
// errLatestVersionChanged reports a delete marker refused by its latestGuard.
|
||||
var errLatestVersionChanged = errors.New("latest version changed")
|
||||
|
||||
// createDeleteMarker creates a delete marker for versioned delete operations
|
||||
func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error) {
|
||||
return s3a.createDeleteMarkerIf(bucket, object, nil)
|
||||
}
|
||||
|
||||
// createDeleteMarkerIf is createDeleteMarker whose routed commit also requires
|
||||
// latestGuard to hold on the .versions directory entry. The unrouted path
|
||||
// has no transaction to check it in and writes unconditionally.
|
||||
func (s3a *S3ApiServer) createDeleteMarkerIf(bucket, object string, latestGuard *filer_pb.WriteCondition) (string, error) {
|
||||
// Clean up the object path first
|
||||
cleanObject := strings.TrimPrefix(object, "/")
|
||||
|
||||
@@ -239,7 +249,11 @@ func (s3a *S3ApiServer) createDeleteMarker(bucket, object string) (string, error
|
||||
},
|
||||
Extended: deleteMarkerExtended,
|
||||
}
|
||||
if code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry); code != s3err.ErrNone {
|
||||
code := s3a.routedVersionedFinalize(owner, bucket, cleanObject, useInvertedFormat, deleteMarkerEntry, latestGuard)
|
||||
if code == s3err.ErrPreconditionFailed {
|
||||
return "", errLatestVersionChanged
|
||||
}
|
||||
if code != s3err.ErrNone {
|
||||
return "", fmt.Errorf("createDeleteMarker: routed finalize failed for %s/%s: code %d", bucket, object, code)
|
||||
}
|
||||
glog.V(2).Infof("createDeleteMarker: successfully created delete marker %s for %s/%s", versionId, bucket, object)
|
||||
|
||||
@@ -51,7 +51,7 @@ func TestPutToFilerStorageClass(t *testing.T) {
|
||||
}
|
||||
filePath, object, dest := "/buckets/b/o", "o", ""
|
||||
if tt.part {
|
||||
r = withMultipartPart(r)
|
||||
r = withMultipartPart(r, s3a.uploadFiler("b", "o"))
|
||||
filePath, object, dest = "/buckets/b/.uploads/u/0001.part", "", "/buckets/b/o"
|
||||
}
|
||||
_, code, _ := s3a.putToFiler(r, filePath, strings.NewReader("hello world"), "b", object, 1, 0, nil, false, dest)
|
||||
|
||||
@@ -52,10 +52,15 @@ type Entry struct {
|
||||
// Identity is the listed entry's fingerprint; the server refuses the
|
||||
// delete when the live entry no longer matches it.
|
||||
Identity *router.EntryIdentity
|
||||
|
||||
// Resume is the ListFunc start value that continues the walk after
|
||||
// this entry. Empty keeps the previous position, as for every
|
||||
// version of an object but its last.
|
||||
Resume string
|
||||
}
|
||||
|
||||
// ListFunc must skip entries with Path <= start so kill-resume picks up
|
||||
// where the previous run stopped.
|
||||
// ListFunc lists the bucket, starting after start: "" or the Resume of
|
||||
// an entry an earlier walk finished.
|
||||
type ListFunc func(ctx context.Context, bucket, start string, cb func(*Entry) error) error
|
||||
|
||||
// Dispatcher executes one (action, entry) verdict. An error halts the walk;
|
||||
@@ -64,8 +69,8 @@ type Dispatcher interface {
|
||||
Delete(ctx context.Context, action *engine.CompiledAction, entry *Entry) error
|
||||
}
|
||||
|
||||
// Checkpoint is the resume state. Caller persists it under
|
||||
// /etc/s3/lifecycle/<bucket>/_bootstrap.
|
||||
// Checkpoint is the resume state: LastScannedPath is the Resume of the
|
||||
// last finished entry, to pass back as WalkOptions.Resume.
|
||||
type Checkpoint struct {
|
||||
LastScannedPath string
|
||||
Completed bool
|
||||
@@ -97,14 +102,14 @@ func Walk(ctx context.Context, snap *engine.Snapshot, bucket string, list ListFu
|
||||
}
|
||||
// MPU init directories at .uploads/<id> are the one directory
|
||||
// shape lifecycle cares about; everything else stays out.
|
||||
if entry.IsDirectory && !entry.IsMPUInit {
|
||||
cp.LastScannedPath = entry.Path
|
||||
return nil
|
||||
if !entry.IsDirectory || entry.IsMPUInit {
|
||||
if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := walkEntry(ctx, snap, bucket, entry, dispatch, now, &info); err != nil {
|
||||
return err
|
||||
if entry.Resume != "" {
|
||||
cp.LastScannedPath = entry.Resume
|
||||
}
|
||||
cp.LastScannedPath = entry.Path
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -184,13 +189,17 @@ func walkEntry(ctx context.Context, snap *engine.Snapshot, bucket string, entry
|
||||
return nil
|
||||
}
|
||||
|
||||
// EntryCallback wraps an in-memory slice as a ListFunc; useful for tests.
|
||||
// EntryCallback wraps an in-memory slice as a ListFunc that resumes by
|
||||
// Path; useful for tests.
|
||||
func EntryCallback(entries []*Entry) ListFunc {
|
||||
return func(ctx context.Context, bucket, start string, cb func(*Entry) error) error {
|
||||
for _, e := range entries {
|
||||
if start != "" && e.Path <= start {
|
||||
continue
|
||||
}
|
||||
if e != nil && e.Resume == "" {
|
||||
e.Resume = e.Path
|
||||
}
|
||||
if err := cb(e); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -429,3 +429,39 @@ func TestWalk_MPUInitDoesNotFireNoncurrent(t *testing.T) {
|
||||
t.Fatalf("kind=%v, want AbortMPU", rec.calls[0].kind)
|
||||
}
|
||||
}
|
||||
|
||||
type failVersion string
|
||||
|
||||
func (f failVersion) Delete(_ context.Context, _ *engine.CompiledAction, entry *Entry) error {
|
||||
if entry.VersionID == string(f) {
|
||||
return errors.New("dispatch boom")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestWalk_CheckpointStaysBeforeUnfinishedVersionGroup(t *testing.T) {
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
||||
snap := compileEvDriven(t, "bk", rule)
|
||||
mod := mustTime(t, "2024-01-01T00:00:00Z")
|
||||
list := func(_ context.Context, _, _ string, cb func(*Entry) error) error {
|
||||
for _, e := range []*Entry{
|
||||
{Path: "a", IsLatest: true, ModTime: mod, Resume: "after-a"},
|
||||
{Path: "obj", VersionID: "v2", IsLatest: true, ModTime: mod},
|
||||
{Path: "obj", VersionID: "v1", ModTime: mod, Resume: "after-obj"},
|
||||
} {
|
||||
if err := cb(e); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
cp, err := Walk(context.Background(), snap, "bk", list, failVersion("v2"), WalkOptions{Now: mod.AddDate(0, 0, 10)})
|
||||
if err == nil || cp.LastScannedPath != "after-a" {
|
||||
t.Fatalf("failure inside the group: got cp=%+v err=%v, want after-a and an error", cp, err)
|
||||
}
|
||||
cp, err = Walk(context.Background(), snap, "bk", list, failVersion("none"), WalkOptions{Now: mod.AddDate(0, 0, 10)})
|
||||
if err != nil || !cp.Completed || cp.LastScannedPath != "after-obj" {
|
||||
t.Fatalf("finished walk: got cp=%+v err=%v, want after-obj", cp, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestDispatch_UnavailableThenSuccess(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "connection error")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestDispatch_ServerDeadlineIsRetried(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.DeadlineExceeded, "context deadline exceeded")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Minute)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestDispatch_PermanentErrorsAreNotRetried(t *testing.T) {
|
||||
for _, code := range []codes.Code{codes.Unauthenticated, codes.PermissionDenied, codes.InvalidArgument, codes.Unimplemented} {
|
||||
t.Run(code.String(), func(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(code, "rejected")}}}
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, code, status.Code(err))
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDispatch_NilResponseIsAnErrorNotAPanic(t *testing.T) {
|
||||
c := &walkerStubClient{nilResp: true}
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 0)
|
||||
require.ErrorIs(t, err, errNilDeleteResponse)
|
||||
}
|
||||
|
||||
func TestDispatch_EveryAttemptWaitsOnLimiter(t *testing.T) {
|
||||
lim := rate.NewLimiter(rate.Every(time.Hour), transportRetryAttempts)
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
_, err := dispatchWithRetry(context.Background(), c, lim, sampleMatch(), 0)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(transportRetryAttempts), c.calls)
|
||||
assert.Less(t, lim.Tokens(), 0.5, "each attempt must take a token from the shared budget")
|
||||
}
|
||||
|
||||
func TestProcessMatches_RetryExhaustionHoldsCursorBeforeTheMatch(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
first, second := sampleMatch(), sampleMatch()
|
||||
first.DueTime, second.DueTime = runNow.Add(-time.Hour), runNow.Add(-time.Hour)
|
||||
second.ObjectKey = "obj2"
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
{err: status.Error(codes.Unavailable, "down")},
|
||||
}}
|
||||
_, halted, err := processMatches(context.Background(), Config{Client: c}, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{first, second})
|
||||
require.Error(t, err)
|
||||
assert.True(t, halted)
|
||||
assert.Equal(t, int32(1+transportRetryAttempts), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_TransientErrorIsRetried(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{
|
||||
{err: status.Error(codes.Unavailable, "connection reset")},
|
||||
{outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE},
|
||||
}}
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int32(2), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_AuthErrorSurfacesAtOnce(t *testing.T) {
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: status.Error(codes.Unauthenticated, "invalid admin token")}}}
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, codes.Unauthenticated, status.Code(err))
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_ParentCancelDuringBackoffStops(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport: connection refused")}}}
|
||||
time.AfterFunc(10*time.Millisecond, cancel)
|
||||
d := &WalkerDispatcher{Client: c}
|
||||
err := d.Delete(ctx, sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, int32(1), c.calls)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/reader"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// stallingClient blocks each attempt until its context ends, except
|
||||
// attempts listed in answerAfter, which answer DONE after that delay.
|
||||
type stallingClient struct {
|
||||
mu sync.Mutex
|
||||
requests []*s3_lifecycle_pb.LifecycleDeleteRequest
|
||||
answerAfter map[int]time.Duration
|
||||
onCall func(attempt int)
|
||||
}
|
||||
|
||||
func (c *stallingClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
c.mu.Lock()
|
||||
c.requests = append(c.requests, req)
|
||||
attempt := len(c.requests)
|
||||
delay, answers := c.answerAfter[attempt]
|
||||
c.mu.Unlock()
|
||||
if c.onCall != nil {
|
||||
c.onCall(attempt)
|
||||
}
|
||||
if answers {
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
return &s3_lifecycle_pb.LifecycleDeleteResponse{Outcome: s3_lifecycle_pb.LifecycleDeleteOutcome_DONE}, nil
|
||||
case <-ctx.Done():
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
}
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
|
||||
func (c *stallingClient) calls() int {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return len(c.requests)
|
||||
}
|
||||
|
||||
func versionedMatch() router.Match {
|
||||
m := sampleMatch()
|
||||
m.Key.ActionKind = s3lifecycle.ActionKindNoncurrentDays
|
||||
m.VersionID = "v_1"
|
||||
m.Identity = &router.EntryIdentity{MtimeNs: 1, Size: 7, HeadFid: "3,01", ExtendedHash: []byte{0xaa}}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestDispatch_StalledAttemptsTimeOutAndFail(t *testing.T) {
|
||||
c := &stallingClient{}
|
||||
start := time.Now()
|
||||
_, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), 20*time.Millisecond)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, context.DeadlineExceeded)
|
||||
assert.Contains(t, err.Error(), "timed out after 20ms")
|
||||
assert.Equal(t, transportRetryAttempts, c.calls(), "an attempt timeout is retried like a transport error")
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestDispatch_SlowAttemptWithinTimeoutSucceeds(t *testing.T) {
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 30 * time.Millisecond}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, sampleMatch(), time.Second)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
assert.Equal(t, 1, c.calls())
|
||||
}
|
||||
|
||||
func TestDispatch_RetryAfterTimeoutResendsSameRequest(t *testing.T) {
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{2: 0}}
|
||||
out, err := dispatchWithRetry(context.Background(), c, nil, versionedMatch(), 20*time.Millisecond)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, s3_lifecycle_pb.LifecycleDeleteOutcome_DONE, out)
|
||||
require.Len(t, c.requests, 2)
|
||||
assert.True(t, proto.Equal(c.requests[0], c.requests[1]), "a retry after a lost reply must keep the version and identity witness")
|
||||
assert.Equal(t, "v_1", c.requests[1].VersionId)
|
||||
assert.Equal(t, "3,01", c.requests[1].ExpectedIdentity.GetHeadFid())
|
||||
}
|
||||
|
||||
func TestDispatch_ParentCancelInterruptsActiveAttempt(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &stallingClient{onCall: func(int) { time.AfterFunc(20*time.Millisecond, cancel) }}
|
||||
start := time.Now()
|
||||
_, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, 1, c.calls(), "parent cancellation must not be retried")
|
||||
assert.Less(t, time.Since(start), 5*time.Second)
|
||||
}
|
||||
|
||||
func TestDispatch_ParentCancelInterruptsBackoff(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
c := &fakeLifecycleClient{scripted: []scriptedResp{{err: errors.New("transport boom")}}}
|
||||
time.AfterFunc(10*time.Millisecond, cancel)
|
||||
_, err := dispatchWithRetry(ctx, c, nil, sampleMatch(), time.Hour)
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, int32(1), c.calls, "cancellation during backoff must stop before the next attempt")
|
||||
}
|
||||
|
||||
func TestProcessMatches_AttemptTimeoutFailsShard(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
m := sampleMatch()
|
||||
m.DueTime = runNow.Add(-time.Hour)
|
||||
c := &stallingClient{}
|
||||
cfg := Config{Client: c, DeleteTimeout: 10 * time.Millisecond}
|
||||
_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m})
|
||||
require.ErrorIs(t, err, context.DeadlineExceeded, "exhausted attempts must fail the pass, not report it clean")
|
||||
assert.True(t, halted, "an unresolved delete must hold the cursor")
|
||||
}
|
||||
|
||||
func TestProcessMatches_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) {
|
||||
runNow := time.Now()
|
||||
m := sampleMatch()
|
||||
m.DueTime = runNow.Add(-time.Hour)
|
||||
lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1)
|
||||
_ = lim.AllowN(time.Now(), 1)
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}}
|
||||
cfg := Config{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond}
|
||||
_, halted, err := processMatches(context.Background(), cfg, runNow, &reader.Event{TsNs: runNow.UnixNano()}, []router.Match{m})
|
||||
require.NoError(t, err)
|
||||
assert.False(t, halted)
|
||||
assert.Equal(t, 1, c.calls())
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_AttemptTimeoutHaltsWalk(t *testing.T) {
|
||||
c := &stallingClient{}
|
||||
d := &WalkerDispatcher{Client: c, DeleteTimeout: 10 * time.Millisecond}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "timed out after 10ms")
|
||||
assert.Equal(t, transportRetryAttempts, c.calls(), "the walker shares the replay retry budget")
|
||||
}
|
||||
|
||||
func TestWalkerDispatcher_LimiterWaitDoesNotSpendDeleteTimeout(t *testing.T) {
|
||||
lim := rate.NewLimiter(rate.Every(150*time.Millisecond), 1)
|
||||
_ = lim.AllowN(time.Now(), 1)
|
||||
c := &stallingClient{answerAfter: map[int]time.Duration{1: 20 * time.Millisecond}}
|
||||
d := &WalkerDispatcher{Client: c, Limiter: lim, DeleteTimeout: 100 * time.Millisecond}
|
||||
err := d.Delete(context.Background(), sampleAction(t, s3lifecycle.ActionKindExpirationDays), &bootstrap.Entry{Path: "obj"})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
@@ -3,19 +3,22 @@ package dailyrun
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/router"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/time/rate"
|
||||
"google.golang.org/grpc/codes"
|
||||
)
|
||||
|
||||
// transportRetryAttempts is the in-run retry budget for transport
|
||||
// errors (RPC dial failures, stream resets). Server-side outcomes
|
||||
// (RETRY_LATER, BLOCKED) are NOT retried in-run — the server already
|
||||
// classified them as "wait until next run." The retry exists only to
|
||||
// paper over a single flake from the network.
|
||||
// transportRetryAttempts is the in-run retry budget shared by the walker
|
||||
// and replay deletes. Only transport failures and attempt timeouts are
|
||||
// retried; server outcomes (RETRY_LATER, BLOCKED) wait for the next run.
|
||||
const transportRetryAttempts = 3
|
||||
|
||||
// transportRetryInitial is the first sleep after a failed attempt.
|
||||
@@ -26,41 +29,89 @@ const (
|
||||
transportRetryMax = 5 * time.Second
|
||||
)
|
||||
|
||||
// dispatchWithRetry sends one LifecycleDelete request, retrying on
|
||||
// transport errors up to transportRetryAttempts times with exponential
|
||||
// backoff. Returns the server's outcome on success, or an error if all
|
||||
// retries exhausted. Server outcomes (RETRY_LATER / BLOCKED) bypass
|
||||
// the retry — the daily run's halt-on-failure semantics handles them.
|
||||
func dispatchWithRetry(ctx context.Context, client LifecycleClient, m router.Match) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
|
||||
req := buildDeleteRequest(m)
|
||||
// DefaultDeleteTimeout bounds one LifecycleDelete attempt when the
|
||||
// caller sets no timeout.
|
||||
const DefaultDeleteTimeout = time.Minute
|
||||
|
||||
var (
|
||||
errDeleteAttemptTimedOut = errors.New("lifecycle delete attempt timed out")
|
||||
errNilDeleteResponse = errors.New("nil response")
|
||||
)
|
||||
|
||||
// deleteOnce sends one LifecycleDelete attempt under its own deadline,
|
||||
// so a stalled RPC fails that attempt instead of holding the shard for
|
||||
// the job's lifetime.
|
||||
func deleteOnce(ctx context.Context, client LifecycleClient, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
if timeout <= 0 {
|
||||
timeout = DefaultDeleteTimeout
|
||||
}
|
||||
attemptCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
resp, err := client.LifecycleDelete(attemptCtx, req)
|
||||
if err != nil && ctx.Err() == nil && attemptCtx.Err() != nil {
|
||||
err = fmt.Errorf("%w after %s: %w", errDeleteAttemptTimedOut, timeout, err)
|
||||
}
|
||||
if err == nil && resp == nil {
|
||||
err = errNilDeleteResponse
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// isRetryableDeleteError reports whether a failed attempt may be resent.
|
||||
// Authentication, permission and argument errors fail the same way
|
||||
// again, so they surface at once.
|
||||
func isRetryableDeleteError(err error) bool {
|
||||
if errors.Is(err, errDeleteAttemptTimedOut) || util.IsTransientError(err) {
|
||||
return true
|
||||
}
|
||||
st, ok := util.ServerStatus(err)
|
||||
return ok && st.Code() == codes.DeadlineExceeded
|
||||
}
|
||||
|
||||
// deleteWithRetry sends req, retrying transport failures and attempt
|
||||
// timeouts up to transportRetryAttempts times with jittered exponential
|
||||
// backoff. Every attempt waits on limiter, so retries stay inside the
|
||||
// shared delete budget. The resent request keeps its version and identity
|
||||
// witness, so a delete that committed before its reply was lost resolves
|
||||
// as NOOP_RESOLVED on the retry.
|
||||
func deleteWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, req *s3_lifecycle_pb.LifecycleDeleteRequest, timeout time.Duration) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
backoff := transportRetryInitial
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= transportRetryAttempts; attempt++ {
|
||||
resp, err := client.LifecycleDelete(ctx, req)
|
||||
for attempt := 1; ; attempt++ {
|
||||
if limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if err := limiter.Wait(ctx); err != nil {
|
||||
return nil, fmt.Errorf("limiter: %w", err)
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
resp, err := deleteOnce(ctx, client, req, timeout)
|
||||
if err == nil {
|
||||
return resp.Outcome, nil
|
||||
return resp, nil
|
||||
}
|
||||
// Context cancellation is shutdown, not a transport flake.
|
||||
// Surface it immediately so the caller halts and tomorrow
|
||||
// resumes from the same cursor.
|
||||
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err
|
||||
// Parent cancellation is shutdown, not a transport flake.
|
||||
if ctxErr := ctx.Err(); ctxErr != nil {
|
||||
return nil, ctxErr
|
||||
}
|
||||
lastErr = err
|
||||
if attempt == transportRetryAttempts {
|
||||
break
|
||||
if attempt >= transportRetryAttempts || !isRetryableDeleteError(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, ctx.Err()
|
||||
return nil, ctx.Err()
|
||||
case <-time.After(jitter(backoff)):
|
||||
}
|
||||
backoff *= 2
|
||||
if backoff > transportRetryMax {
|
||||
backoff = transportRetryMax
|
||||
}
|
||||
backoff = min(backoff*2, transportRetryMax)
|
||||
}
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, lastErr
|
||||
}
|
||||
|
||||
// dispatchWithRetry sends the LifecycleDelete for m under the shared
|
||||
// retry policy and returns the server's outcome.
|
||||
func dispatchWithRetry(ctx context.Context, client LifecycleClient, limiter *rate.Limiter, m router.Match, timeout time.Duration) (s3_lifecycle_pb.LifecycleDeleteOutcome, error) {
|
||||
resp, err := deleteWithRetry(ctx, client, limiter, buildDeleteRequest(m), timeout)
|
||||
if err != nil {
|
||||
return s3_lifecycle_pb.LifecycleDeleteOutcome_LIFECYCLE_DELETE_OUTCOME_UNSPECIFIED, err
|
||||
}
|
||||
return resp.Outcome, nil
|
||||
}
|
||||
|
||||
// jitter returns a duration in the range [d/2, d) using equal jitter.
|
||||
|
||||
@@ -56,7 +56,7 @@ func dispatchWithRetryFast(t *testing.T, ctx context.Context, c LifecycleClient,
|
||||
// We can't override the package-private constants, but every test
|
||||
// path here uses small attempt counts so even the production
|
||||
// backoff is fine.
|
||||
return dispatchWithRetry(ctx, c, m)
|
||||
return dispatchWithRetry(ctx, c, nil, m, 0)
|
||||
}
|
||||
|
||||
func TestDispatch_FirstAttemptSucceeds(t *testing.T) {
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
@@ -39,6 +40,13 @@ func init() { listPageSize.Store(1024) }
|
||||
// active rule prefix can reach are pruned rather than listed. A nil
|
||||
// view assumes the worst (versioned, unpruned), preserving the
|
||||
// two-pass behavior.
|
||||
//
|
||||
// start and Entry.Resume are walk positions: one "<pass>:<name>" frame
|
||||
// per directory level, joined by "/", naming the last finished entry.
|
||||
// They follow the traversal, which is not name order: at every level
|
||||
// pass 1 (.versions groups) precedes pass 2, and a directory's children
|
||||
// come before a sibling file such as "dir.txt". A version group is one
|
||||
// unit, so a resume repeats it until all of its versions finished.
|
||||
func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view *engine.Snapshot) bootstrap.ListFunc {
|
||||
return func(ctx context.Context, bucket, start string, cb func(*bootstrap.Entry) error) error {
|
||||
if client == nil {
|
||||
@@ -53,10 +61,47 @@ func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view
|
||||
return nil // no active rule can match anything in this bucket
|
||||
}
|
||||
}
|
||||
return walkBucketTree(ctx, client, root, root, start, versioned, prefixes, cb)
|
||||
at, err := parseWalkPosition(start)
|
||||
if err != nil {
|
||||
glog.Warningf("lifecycle walk %s: %v; walking the whole bucket", bucket, err)
|
||||
}
|
||||
return walkBucketTree(ctx, client, root, root, "", at, versioned, prefixes, cb)
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
passVersions byte = '1'
|
||||
passEntries byte = '2'
|
||||
)
|
||||
|
||||
type walkFrame struct {
|
||||
pass byte
|
||||
name string
|
||||
}
|
||||
|
||||
func walkPosition(parent string, pass byte, name string) string {
|
||||
frame := string(pass) + ":" + name
|
||||
if parent == "" {
|
||||
return frame
|
||||
}
|
||||
return parent + "/" + frame
|
||||
}
|
||||
|
||||
func parseWalkPosition(pos string) ([]walkFrame, error) {
|
||||
if pos == "" {
|
||||
return nil, nil
|
||||
}
|
||||
parts := strings.Split(pos, "/")
|
||||
frames := make([]walkFrame, len(parts))
|
||||
for i, p := range parts {
|
||||
if len(p) < 3 || p[1] != ':' || (p[0] != passVersions && p[0] != passEntries) {
|
||||
return nil, fmt.Errorf("bad walk position %q", pos)
|
||||
}
|
||||
frames[i] = walkFrame{pass: p[0], name: p[2:]}
|
||||
}
|
||||
return frames, nil
|
||||
}
|
||||
|
||||
// walkBucketTree recurses through dir in two passes per level. Pass 1
|
||||
// expands `.versions/` dirs (populating skipBare with the bare null-
|
||||
// version keys that pass 2 must suppress) and only runs when the bucket
|
||||
@@ -67,12 +112,27 @@ func FilerListFunc(client filer_pb.SeaweedFilerClient, bucketsPath string, view
|
||||
// `.versions/` dirs must be processed before their bare sibling so the
|
||||
// walk sees whether the bare entry is the null version of the same
|
||||
// logical object; hence the two passes on versioned buckets.
|
||||
func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot, start string, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error {
|
||||
//
|
||||
// pos names dir's position; at is what is left of the resume position
|
||||
// below it.
|
||||
func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, bucketRoot, pos string, at []walkFrame, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error {
|
||||
skipBare := map[string]bool{}
|
||||
var here walkFrame
|
||||
var below []walkFrame
|
||||
resuming := len(at) > 0
|
||||
if resuming {
|
||||
here, below = at[0], at[1:]
|
||||
}
|
||||
|
||||
// Pass 1: .versions/ dirs only.
|
||||
if versioned {
|
||||
if err := listAll(ctx, client, dir, func(e *filer_pb.Entry) error {
|
||||
// From pass 2, only the groups whose bare file is still ahead
|
||||
// matter, and they sort after here.name.
|
||||
from := ""
|
||||
if resuming && here.pass == passEntries {
|
||||
from = here.name
|
||||
}
|
||||
if err := listAll(ctx, client, dir, from, false, func(e *filer_pb.Entry) error {
|
||||
if e == nil || e.Attributes == nil {
|
||||
return nil
|
||||
}
|
||||
@@ -87,17 +147,42 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir
|
||||
if !dirInRuleScope(prefixes, key) {
|
||||
return nil
|
||||
}
|
||||
return expandVersionsDir(ctx, client, bucketRoot, key, e, start, skipBare, versioned, prefixes, cb)
|
||||
var inner []walkFrame
|
||||
if resuming {
|
||||
if here.pass == passEntries || e.Name < here.name || (e.Name == here.name && len(below) == 0) {
|
||||
// Finished earlier. Its bare null version still has
|
||||
// to stay out of pass 2; for a group that had no
|
||||
// versions this defers the bare file to the next walk.
|
||||
skipBare[strings.TrimSuffix(key, s3_constants.VersionsFolder)] = true
|
||||
return nil
|
||||
}
|
||||
if e.Name == here.name {
|
||||
inner = below
|
||||
}
|
||||
}
|
||||
return expandVersionsDir(ctx, client, bucketRoot, key, e, walkPosition(pos, passVersions, e.Name), inner, skipBare, versioned, prefixes, cb)
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Pass 2: everything else.
|
||||
return listAll(ctx, client, dir, func(e *filer_pb.Entry) error {
|
||||
from, inclusive := "", false
|
||||
if resuming && here.pass == passEntries {
|
||||
from, inclusive = here.name, true
|
||||
}
|
||||
return listAll(ctx, client, dir, from, inclusive, func(e *filer_pb.Entry) error {
|
||||
if e == nil || e.Attributes == nil {
|
||||
return nil
|
||||
}
|
||||
var inner []walkFrame
|
||||
if resuming && here.pass == passEntries && e.Name == here.name {
|
||||
if len(below) == 0 {
|
||||
return nil
|
||||
}
|
||||
inner = below
|
||||
}
|
||||
entryPos := walkPosition(pos, passEntries, e.Name)
|
||||
full := dir + "/" + e.Name
|
||||
key := strings.TrimPrefix(full, bucketRoot+"/")
|
||||
if e.IsDirectory && isVersionsDir(e) {
|
||||
@@ -109,13 +194,10 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir
|
||||
if !dirInRuleScope(prefixes, key) {
|
||||
return nil
|
||||
}
|
||||
return walkBucketTree(ctx, client, full, bucketRoot, start, false, prefixes, cb)
|
||||
return walkBucketTree(ctx, client, full, bucketRoot, entryPos, inner, false, prefixes, cb)
|
||||
}
|
||||
if e.IsDirectory {
|
||||
if isMPUInitDir(key, e) {
|
||||
if start != "" && key <= start {
|
||||
return nil
|
||||
}
|
||||
destKey := string(e.Extended[s3_constants.ExtMultipartObjectKey])
|
||||
return cb(&bootstrap.Entry{
|
||||
Path: key,
|
||||
@@ -123,6 +205,7 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir
|
||||
IsMPUInit: true,
|
||||
ModTime: time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)),
|
||||
Size: int64(e.Attributes.FileSize),
|
||||
Resume: entryPos,
|
||||
})
|
||||
}
|
||||
// .uploads is exempt: an AbortMPU rule's prefix applies to
|
||||
@@ -130,14 +213,11 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir
|
||||
if key != s3_constants.MultipartUploadsFolder && !dirInRuleScope(prefixes, key) {
|
||||
return nil
|
||||
}
|
||||
return walkBucketTree(ctx, client, full, bucketRoot, start, versioned, prefixes, cb)
|
||||
return walkBucketTree(ctx, client, full, bucketRoot, entryPos, inner, versioned, prefixes, cb)
|
||||
}
|
||||
if skipBare[key] {
|
||||
return nil
|
||||
}
|
||||
if start != "" && key <= start {
|
||||
return nil
|
||||
}
|
||||
entry := &bootstrap.Entry{
|
||||
Path: key,
|
||||
ModTime: time.Unix(e.Attributes.Mtime, int64(e.Attributes.MtimeNs)),
|
||||
@@ -145,6 +225,7 @@ func walkBucketTree(ctx context.Context, client filer_pb.SeaweedFilerClient, dir
|
||||
IsLatest: true, // Non-versioned default.
|
||||
Tags: extractTags(e.Extended),
|
||||
Identity: router.BuildIdentityFromEntry(e),
|
||||
Resume: entryPos,
|
||||
}
|
||||
return cb(entry)
|
||||
})
|
||||
@@ -171,18 +252,16 @@ type versionItem struct {
|
||||
// streaming and walker paths reach the same verdict for the same
|
||||
// objects. Phase 5 deletes the scheduler copy.
|
||||
//
|
||||
// Resume note: every emitted sibling shares Path = logical key, so a
|
||||
// resume after a mid-expansion failure rewalks the whole sibling
|
||||
// group. Acceptable today because Phase 4b doesn't persist a
|
||||
// Checkpoint between runs (start is always "" via runShard).
|
||||
func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, versionsKey string, versionsEntry *filer_pb.Entry, start string, skipBare map[string]bool, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error {
|
||||
// Only the last version carries the group's position pos, so a walk
|
||||
// that stops inside the group resumes before it.
|
||||
func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketRoot, versionsKey string, versionsEntry *filer_pb.Entry, pos string, at []walkFrame, skipBare map[string]bool, versioned bool, prefixes []string, cb func(*bootstrap.Entry) error) error {
|
||||
logical := strings.TrimSuffix(versionsKey, s3_constants.VersionsFolder)
|
||||
if logical == "" {
|
||||
return nil
|
||||
}
|
||||
versionsDir := bucketRoot + "/" + versionsKey
|
||||
var children []*filer_pb.Entry
|
||||
if err := listAll(ctx, client, versionsDir, func(e *filer_pb.Entry) error {
|
||||
if err := listAll(ctx, client, versionsDir, "", false, func(e *filer_pb.Entry) error {
|
||||
if e != nil && e.Attributes != nil && !e.IsDirectory {
|
||||
children = append(children, e)
|
||||
}
|
||||
@@ -200,7 +279,7 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient,
|
||||
// Coincidentally-named user folder, or an empty `.versions`
|
||||
// container. Treat it as a regular subdirectory so user-named
|
||||
// files inside still surface.
|
||||
return walkBucketTree(ctx, client, versionsDir, bucketRoot, start, versioned, prefixes, cb)
|
||||
return walkBucketTree(ctx, client, versionsDir, bucketRoot, pos, at, versioned, prefixes, cb)
|
||||
}
|
||||
if nullEntry, nullKey, explicit, ok := lookupNullVersion(ctx, client, bucketRoot, logical); ok {
|
||||
items = append(items, versionItem{
|
||||
@@ -253,11 +332,6 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient,
|
||||
}
|
||||
}
|
||||
|
||||
if start != "" && logical <= start {
|
||||
// All siblings share Path=logical, so the whole group is
|
||||
// either above or below the resume marker.
|
||||
return nil
|
||||
}
|
||||
for i, it := range items {
|
||||
successor := s3lifecycle.SuccessorFromEntryStamp(it.entry)
|
||||
if successor.IsZero() && i > 0 {
|
||||
@@ -277,6 +351,9 @@ func expandVersionsDir(ctx context.Context, client filer_pb.SeaweedFilerClient,
|
||||
Tags: extractTags(it.entry.Extended),
|
||||
Identity: router.BuildIdentityFromEntry(it.entry),
|
||||
}
|
||||
if i == len(items)-1 {
|
||||
entry.Resume = pos
|
||||
}
|
||||
if !isLatest {
|
||||
rank := i
|
||||
if i > latestPos {
|
||||
@@ -323,9 +400,8 @@ func lookupNullVersion(ctx context.Context, client filer_pb.SeaweedFilerClient,
|
||||
// listAll issues paginated SeaweedList calls until exhausted. Ported
|
||||
// from scheduler/bootstrap.go's same-named helper; Phase 5 deletes
|
||||
// the scheduler copy when the streaming path is removed.
|
||||
func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string, fn func(*filer_pb.Entry) error) error {
|
||||
func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir, startFrom string, inclusive bool, fn func(*filer_pb.Entry) error) error {
|
||||
pageSize := listPageSize.Load()
|
||||
startFrom := ""
|
||||
for {
|
||||
var pageCount uint32
|
||||
var lastName string
|
||||
@@ -335,13 +411,13 @@ func listAll(ctx context.Context, client filer_pb.SeaweedFilerClient, dir string
|
||||
lastName = e.Name
|
||||
}
|
||||
return fn(e)
|
||||
}, startFrom, false, pageSize); err != nil {
|
||||
}, startFrom, inclusive, pageSize); err != nil {
|
||||
return err
|
||||
}
|
||||
if pageCount < pageSize {
|
||||
return nil
|
||||
}
|
||||
startFrom = lastName
|
||||
startFrom, inclusive = lastName, false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ func (c *fakeFiler) ListEntries(ctx context.Context, in *filer_pb.ListEntriesReq
|
||||
if e == nil {
|
||||
continue
|
||||
}
|
||||
if in.StartFromFileName != "" && !in.InclusiveStartFrom && e.Name <= in.StartFromFileName {
|
||||
if in.StartFromFileName != "" && (e.Name < in.StartFromFileName || (!in.InclusiveStartFrom && e.Name == in.StartFromFileName)) {
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, e)
|
||||
@@ -461,18 +461,105 @@ func TestFilerListFunc_VersionedDeleteMarkerPropagates(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestFilerListFunc_HonorsStart(t *testing.T) {
|
||||
// The walker's kill-resume contract: skip entries whose Path <= start.
|
||||
mtime := time.Now()
|
||||
client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
|
||||
"/buckets/bkt": {file("a", mtime, 1), file("b", mtime, 1), file("c", mtime, 1)},
|
||||
}}
|
||||
listFn := FilerListFunc(client, "/buckets", nil)
|
||||
var got []string
|
||||
require.NoError(t, listFn(context.Background(), "bkt", "a", func(e *bootstrap.Entry) error {
|
||||
require.NoError(t, listFn(context.Background(), "bkt", "2:a", func(e *bootstrap.Entry) error {
|
||||
got = append(got, e.Path)
|
||||
return nil
|
||||
}))
|
||||
assert.Equal(t, []string{"b", "c"}, got)
|
||||
|
||||
got = nil
|
||||
require.NoError(t, listFn(context.Background(), "bkt", "a", func(e *bootstrap.Entry) error {
|
||||
got = append(got, e.Path)
|
||||
return nil
|
||||
}))
|
||||
assert.Equal(t, []string{"a", "b", "c"}, got, "an unreadable position walks the whole bucket")
|
||||
}
|
||||
|
||||
func resumeTestTree() *fakeFiler {
|
||||
t1 := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
t2 := t1.Add(time.Hour)
|
||||
ver := func(id string, mtime time.Time) *filer_pb.Entry {
|
||||
return fileWithExt(id, mtime, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)})
|
||||
}
|
||||
mpu := dir("u1")
|
||||
mpu.Extended = map[string][]byte{s3_constants.ExtMultipartObjectKey: []byte("foo/big")}
|
||||
vs := s3_constants.VersionsFolder
|
||||
return &fakeFiler{tree: map[string][]*filer_pb.Entry{
|
||||
"/buckets/bkt": {
|
||||
dir(s3_constants.MultipartUploadsFolder),
|
||||
file("alpha", t1, 1),
|
||||
dir("foo"),
|
||||
file("foo.txt", t1, 1),
|
||||
fileWithExt("zeta", t2, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte("null")}),
|
||||
versionsDir("zeta"+vs, ""),
|
||||
file("zz", t1, 1),
|
||||
},
|
||||
"/buckets/bkt/" + s3_constants.MultipartUploadsFolder: {mpu},
|
||||
"/buckets/bkt/zeta" + vs: {ver("z1", t1)},
|
||||
"/buckets/bkt/foo": {
|
||||
versionsDir("v"+vs, "v2"),
|
||||
file("x", t1, 1),
|
||||
},
|
||||
"/buckets/bkt/foo/v" + vs: {ver("v1", t1), ver("v2", t2)},
|
||||
}}
|
||||
}
|
||||
|
||||
type listedEntry struct{ Path, VersionID, Resume string }
|
||||
|
||||
func listFrom(t *testing.T, listFn bootstrap.ListFunc, start string) []listedEntry {
|
||||
t.Helper()
|
||||
var got []listedEntry
|
||||
require.NoError(t, listFn(context.Background(), "bkt", start, func(e *bootstrap.Entry) error {
|
||||
got = append(got, listedEntry{e.Path, e.VersionID, e.Resume})
|
||||
return nil
|
||||
}))
|
||||
return got
|
||||
}
|
||||
|
||||
func TestFilerListFunc_ResumeFollowsTraversalOrder(t *testing.T) {
|
||||
prev := listPageSize.Load()
|
||||
listPageSize.Store(2)
|
||||
defer listPageSize.Store(prev)
|
||||
|
||||
client := resumeTestTree()
|
||||
listFn := FilerListFunc(client, "/buckets", nil)
|
||||
full := listFrom(t, listFn, "")
|
||||
fullCalls := client.listCalls
|
||||
var order []string
|
||||
for _, e := range full {
|
||||
order = append(order, e.Path+"@"+e.VersionID)
|
||||
}
|
||||
// Pass 1 groups come before pass 2 at each level, and foo's
|
||||
// children before foo.txt; the null version is part of zeta's group.
|
||||
require.Equal(t, []string{
|
||||
"zeta@null", "zeta@z1",
|
||||
".uploads/u1@",
|
||||
"alpha@",
|
||||
"foo/v@v2", "foo/v@v1",
|
||||
"foo/x@",
|
||||
"foo.txt@",
|
||||
"zz@",
|
||||
}, order)
|
||||
|
||||
for i, e := range full {
|
||||
if e.Resume == "" {
|
||||
continue
|
||||
}
|
||||
assert.Equal(t, append([]listedEntry(nil), full[i+1:]...), listFrom(t, listFn, e.Resume), "resume after %s", e.Resume)
|
||||
}
|
||||
assert.Equal(t, "", full[0].Resume, "only a group's last version moves the position")
|
||||
assert.Equal(t, "1:zeta.versions", full[1].Resume)
|
||||
assert.Equal(t, "2:foo/1:v.versions", full[5].Resume)
|
||||
|
||||
client.listCalls = 0
|
||||
listFrom(t, listFn, "2:foo.txt")
|
||||
assert.Less(t, client.listCalls, fullCalls/2, "a resume skips listing what it already finished")
|
||||
}
|
||||
|
||||
func TestFilerListFunc_NilClient(t *testing.T) {
|
||||
|
||||
@@ -29,8 +29,9 @@ type LifecycleClient interface {
|
||||
|
||||
// WalkerFunc runs one bucket walk covering every shard in shards under
|
||||
// the given engine view, so a whole pass costs at most a couple of
|
||||
// bucket listings rather than one listing per shard.
|
||||
type WalkerFunc func(ctx context.Context, view *engine.Snapshot, shards []int) error
|
||||
// bucket listings rather than one listing per shard. It returns when the
|
||||
// earliest bucket it covers was listed (see WalkBuckets).
|
||||
type WalkerFunc func(ctx context.Context, view *engine.Snapshot, shards []int) (time.Time, error)
|
||||
|
||||
type Config struct {
|
||||
Shards []int
|
||||
@@ -47,6 +48,10 @@ type Config struct {
|
||||
// nil -> no rate limit. Shared across all shard goroutines.
|
||||
Limiter *rate.Limiter
|
||||
|
||||
// DeleteTimeout bounds each LifecycleDelete attempt, applied after
|
||||
// the limiter wait. <= 0 uses DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
|
||||
// Meta-log retention boundary. Rules whose effective TTL exceeds
|
||||
// this can't be serviced by replay alone and get partitioned into
|
||||
// the walk view (engine.PromotedHash). 0 falls back to maxTTL,
|
||||
@@ -561,7 +566,7 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim
|
||||
// the bug TestLifecycleExpirationDateInThePast caught.
|
||||
if cfg.Walker != nil && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) {
|
||||
if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 {
|
||||
if werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil {
|
||||
if _, werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil {
|
||||
return fmt.Errorf("shard=%d: steady walk (empty replay): %w", shardID, werr)
|
||||
}
|
||||
lastWalkedNs = runNow.UnixNano()
|
||||
@@ -590,20 +595,29 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim
|
||||
// via BucketBootstrapper; daily-replay needs the same.
|
||||
mustWalkRecovery := found && (persisted.RuleSetHash != rsh || persisted.PromotedHash != promoted)
|
||||
mustWalkColdStart := !found
|
||||
// A resumed walk listed its earlier buckets before this pass; objects
|
||||
// there that were not due yet must stay inside the replay window.
|
||||
replayFrom := runNow
|
||||
if mustWalkRecovery || mustWalkColdStart {
|
||||
if cfg.Walker != nil {
|
||||
if werr := walkForShard(ctx, cfg.Walker, walks, shardID, true, engine.RecoveryView(snap)); werr != nil {
|
||||
since, werr := walkForShard(ctx, cfg.Walker, walks, shardID, true, engine.RecoveryView(snap))
|
||||
if werr != nil {
|
||||
return fmt.Errorf("shard=%d: recovery walk: %w", shardID, werr)
|
||||
}
|
||||
if !since.IsZero() && since.Before(replayFrom) {
|
||||
replayFrom = since
|
||||
}
|
||||
lastWalkedNs = runNow.UnixNano()
|
||||
walkedThisPass = true
|
||||
}
|
||||
if mustWalkRecovery {
|
||||
// The shared subscription began at runNow - maxTTL, so a cold
|
||||
// start whose walk resumed an earlier attempt replays next pass.
|
||||
if mustWalkRecovery || replayFrom.Before(runNow) {
|
||||
// Rule changed: rewind cursor so the sliding replay re-scans
|
||||
// the new max-TTL window and the persisted hashes match the
|
||||
// new rule set.
|
||||
next := Cursor{
|
||||
TsNs: runNow.Add(-maxTTL).UnixNano(),
|
||||
TsNs: replayFrom.Add(-maxTTL).UnixNano(),
|
||||
RuleSetHash: rsh,
|
||||
PromotedHash: promoted,
|
||||
LastWalkedNs: lastWalkedNs,
|
||||
@@ -627,7 +641,7 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim
|
||||
// where this is a normal pass that didn't trigger cold-start.
|
||||
if cfg.Walker != nil && !walkedThisPass && walkerDue(lastWalkedNs, runNow, cfg.WalkerInterval) {
|
||||
if _, walkView := snap.RulesForShard(shardID, retentionWindow); walkView != nil && len(walkView.AllActions()) > 0 {
|
||||
if werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil {
|
||||
if _, werr := walkForShard(ctx, cfg.Walker, walks, shardID, false, walkView); werr != nil {
|
||||
return fmt.Errorf("shard=%d: steady walk: %w", shardID, werr)
|
||||
}
|
||||
lastWalkedNs = runNow.UnixNano()
|
||||
@@ -665,8 +679,9 @@ func runShard(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow tim
|
||||
if drainErr != nil {
|
||||
_ = saveCursorAndPublish(saveCtx, cfg.Persister, shardID, Cursor{TsNs: lastOK, RuleSetHash: rsh, PromotedHash: promoted, LastWalkedNs: lastWalkedNs})
|
||||
// Cut short by its wall-clock cap: truncated, not failed — the
|
||||
// cursor above carries progress forward.
|
||||
if errors.Is(drainErr, context.DeadlineExceeded) || errors.Is(drainErr, context.Canceled) {
|
||||
// cursor above carries progress forward. A timed-out delete
|
||||
// attempt also wraps DeadlineExceeded, so judge by ctx itself.
|
||||
if ctx.Err() != nil {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("shard=%d: drain: %w", shardID, drainErr)
|
||||
@@ -797,14 +812,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade
|
||||
skippedAny = true
|
||||
continue
|
||||
}
|
||||
if cfg.Limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if waitErr := cfg.Limiter.Wait(ctx); waitErr != nil {
|
||||
return skippedAny, true, waitErr
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, m)
|
||||
outcome, dispatchErr := dispatchWithRetry(ctx, cfg.Client, cfg.Limiter, m, cfg.DeleteTimeout)
|
||||
if dispatchErr != nil {
|
||||
glog.V(1).Infof("daily_run: transport error on %s/%s %s: %v",
|
||||
m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr)
|
||||
@@ -812,7 +820,7 @@ func processMatches(ctx context.Context, cfg Config, runNow time.Time, ev *reade
|
||||
// (dispatcher/dispatcher.go) so transport failures
|
||||
// aggregate under one outcome key across paths.
|
||||
countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), "RPC_ERROR")
|
||||
return skippedAny, true, nil
|
||||
return skippedAny, true, fmt.Errorf("%s/%s %s: %w", m.Bucket, m.ObjectKey, m.Key.ActionKind, dispatchErr)
|
||||
}
|
||||
countDispatch(cfg.Stats, m.Bucket, m.Key.ActionKind.String(), outcome.String())
|
||||
switch outcome {
|
||||
|
||||
@@ -225,7 +225,7 @@ func TestValidate_RejectsRetentionWindowWithoutWalker(t *testing.T) {
|
||||
cfg := validatableConfig()
|
||||
cfg.RetentionWindow = 24 * time.Hour
|
||||
require.ErrorContains(t, validate(cfg), "RetentionWindow")
|
||||
cfg.Walker = func(context.Context, *engine.Snapshot, []int) error { return nil }
|
||||
cfg.Walker = func(context.Context, *engine.Snapshot, []int) (time.Time, error) { return time.Time{}, nil }
|
||||
require.NoError(t, validate(cfg))
|
||||
cfg.Walker = nil
|
||||
cfg.RetentionWindow = 0
|
||||
|
||||
@@ -4,6 +4,8 @@ import (
|
||||
"context"
|
||||
"io"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -154,3 +156,90 @@ func TestRun_HaltedShardDoesNotStarveOthers(t *testing.T) {
|
||||
assert.Zero(t, sum.ShardErrors)
|
||||
assert.Equal(t, 1, sum.ShardsHalted, "a BLOCKED delete halts its shard without an error")
|
||||
}
|
||||
|
||||
// flooredSubscribeClient serves only the events past the request's
|
||||
// SinceNs, as the filer does, and records each floor it was asked for.
|
||||
type flooredSubscribeClient struct {
|
||||
filer_pb.SeaweedFilerClient
|
||||
mu sync.Mutex
|
||||
responses []*filer_pb.SubscribeMetadataResponse
|
||||
floors []int64
|
||||
}
|
||||
|
||||
func (c *flooredSubscribeClient) SubscribeMetadata(_ context.Context, req *filer_pb.SubscribeMetadataRequest, _ ...grpc.CallOption) (filer_pb.SeaweedFiler_SubscribeMetadataClient, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
c.floors = append(c.floors, req.SinceNs)
|
||||
var served []*filer_pb.SubscribeMetadataResponse
|
||||
for _, r := range c.responses {
|
||||
if r.TsNs > req.SinceNs {
|
||||
served = append(served, r)
|
||||
}
|
||||
}
|
||||
return &scriptedSubscribeStream{responses: served}, nil
|
||||
}
|
||||
|
||||
func TestRun_ColdStartResumedWalkReplaysOlderEventNextPass(t *testing.T) {
|
||||
const bucket = "bk"
|
||||
const shard = 2
|
||||
rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1}
|
||||
prior := map[s3lifecycle.ActionKey]engine.PriorState{}
|
||||
ruleHash := s3lifecycle.RuleHash(rule)
|
||||
for _, k := range s3lifecycle.RuleActionKinds(rule) {
|
||||
prior[s3lifecycle.ActionKey{Bucket: bucket, RuleHash: ruleHash, ActionKind: k}] = engine.PriorState{
|
||||
BootstrapComplete: true,
|
||||
Mode: engine.ModeEventDriven,
|
||||
}
|
||||
}
|
||||
e := engine.New()
|
||||
snap := e.Compile([]engine.CompileInput{{Bucket: bucket, Rules: []*s3lifecycle.Rule{rule}}},
|
||||
engine.CompileOptions{PriorStates: prior})
|
||||
maxTTL := engine.MaxEffectiveTTL(snap)
|
||||
|
||||
runNow := time.Now().UTC()
|
||||
walkedFrom := runNow.Add(-3 * maxTTL)
|
||||
evTime := runNow.Add(-2 * maxTTL)
|
||||
key := keyForShard(t, bucket, shard)
|
||||
laterKey := ""
|
||||
for i := 0; laterKey == ""; i++ {
|
||||
if k := "later" + strconv.Itoa(i); s3lifecycle.ShardID(bucket, k) == shard {
|
||||
laterKey = k
|
||||
}
|
||||
}
|
||||
// The later event is inside the cold-start window; draining it in the
|
||||
// first pass would move the cursor past the older one for good.
|
||||
filer := &flooredSubscribeClient{responses: []*filer_pb.SubscribeMetadataResponse{
|
||||
subscribeResponse(bucket, key, evTime, evTime.UnixNano()),
|
||||
subscribeResponse(bucket, laterKey, evTime, runNow.Add(-maxTTL/2).UnixNano()),
|
||||
}}
|
||||
client := &recordingClient{}
|
||||
var walks atomic.Int32
|
||||
cfg := Config{
|
||||
Shards: []int{shard},
|
||||
BucketsPath: "/buckets",
|
||||
Engine: e,
|
||||
FilerClient: filer,
|
||||
Client: client,
|
||||
Persister: newMemPersister(),
|
||||
Lister: stubSiblingLister{},
|
||||
Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) {
|
||||
if walks.Add(1) == 1 {
|
||||
return walkedFrom, nil
|
||||
}
|
||||
return time.Time{}, nil
|
||||
},
|
||||
}
|
||||
|
||||
cfg.Now = func() time.Time { return runNow }
|
||||
require.NoError(t, Run(context.Background(), cfg))
|
||||
assert.Empty(t, client.seenObjects(), "a cold start that resumed a walk leaves its window to the next pass")
|
||||
|
||||
cfg.Now = func() time.Time { return runNow.Add(time.Minute) }
|
||||
require.NoError(t, Run(context.Background(), cfg))
|
||||
require.Len(t, filer.floors, 2)
|
||||
assert.Equal(t, runNow.Add(-maxTTL).UnixNano(), filer.floors[0])
|
||||
assert.Equal(t, walkedFrom.Add(-maxTTL).UnixNano(), filer.floors[1],
|
||||
"the next pass subscribes from the resumed walk start")
|
||||
assert.Contains(t, client.seenObjects(), key,
|
||||
"the event older than the cold-start floor is replayed on the next pass")
|
||||
}
|
||||
@@ -33,8 +33,8 @@ func TestWalkBuckets_RecordsEveryBucketFailure(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
runStats := &RunStats{}
|
||||
err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "good", 7), []int{0},
|
||||
[]string{"bad-a", "good", "bad-b"}, list, &recordingDispatcher{}, runStats)
|
||||
_, err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "good", 7), []int{0},
|
||||
[]string{"bad-a", "good", "bad-b"}, list, &recordingDispatcher{}, runStats, nil)
|
||||
require.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "bad-a", "the returned error stays the first bucket's")
|
||||
|
||||
@@ -61,8 +61,8 @@ func TestWalkBuckets_CancelMarksRemainingBucketsSkipped(t *testing.T) {
|
||||
return nil
|
||||
}
|
||||
runStats := &RunStats{}
|
||||
err := WalkBuckets(ctx, snapshotForBucketRule(t, "first", 7), []int{0},
|
||||
[]string{"first", "second", "third"}, list, &recordingDispatcher{}, runStats)
|
||||
_, err := WalkBuckets(ctx, snapshotForBucketRule(t, "first", 7), []int{0},
|
||||
[]string{"first", "second", "third"}, list, &recordingDispatcher{}, runStats, nil)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
|
||||
sum := runStats.Summary()
|
||||
@@ -110,8 +110,9 @@ func TestWalkBuckets_CountsEntriesAndObjects(t *testing.T) {
|
||||
{Path: ".uploads/u1", DestKey: "c", IsDirectory: true, IsMPUInit: true, ModTime: old},
|
||||
}
|
||||
runStats := &RunStats{}
|
||||
require.NoError(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "bkt", 7), allShards(),
|
||||
[]string{"bkt"}, bootstrap.EntryCallback(entries), &recordingDispatcher{}, runStats))
|
||||
_, err := WalkBuckets(context.Background(), snapshotForBucketRule(t, "bkt", 7), allShards(),
|
||||
[]string{"bkt"}, bootstrap.EntryCallback(entries), &recordingDispatcher{}, runStats, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
sum := runStats.Summary()
|
||||
assert.Equal(t, int64(5), sum.EntriesVisited, "versions and upload records count, plain directories do not")
|
||||
@@ -169,9 +170,9 @@ func TestRunStats_NilIsANoOp(t *testing.T) {
|
||||
assert.Equal(t, RunSummary{}, runStats.Summary())
|
||||
}
|
||||
|
||||
// A walk can read a bucket without visiting anything: the versioned
|
||||
// pre-pass reads plain files without emitting them, and a resumed walk
|
||||
// skips every key up to its start.
|
||||
// A walk reads entries it does not visit: the versioned pre-pass reads
|
||||
// plain files without emitting them, and a resumed walk re-reads its
|
||||
// start entry.
|
||||
func TestRunStats_CountListedCountsSkippedEntries(t *testing.T) {
|
||||
mtime := time.Now().Add(-7 * 24 * time.Hour)
|
||||
client := &fakeFiler{tree: map[string][]*filer_pb.Entry{
|
||||
@@ -179,13 +180,13 @@ func TestRunStats_CountListedCountsSkippedEntries(t *testing.T) {
|
||||
}}
|
||||
stats := &RunStats{}
|
||||
emitted := 0
|
||||
require.NoError(t, FilerListFunc(stats.CountListed(client), "/buckets", nil)(context.Background(), "bkt", "c.txt", func(*bootstrap.Entry) error {
|
||||
require.NoError(t, FilerListFunc(stats.CountListed(client), "/buckets", nil)(context.Background(), "bkt", "2:b.txt", func(*bootstrap.Entry) error {
|
||||
emitted++
|
||||
return nil
|
||||
}))
|
||||
|
||||
assert.Zero(t, emitted)
|
||||
assert.Equal(t, int64(6), stats.Summary().EntriesListed, "both passes over the versioned bucket")
|
||||
assert.Equal(t, 1, emitted)
|
||||
assert.Equal(t, int64(3), stats.Summary().EntriesListed, "c.txt in the pre-pass, b.txt and c.txt in pass 2")
|
||||
var nilStats *RunStats
|
||||
assert.Same(t, client, nilStats.CountListed(client))
|
||||
}
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
@@ -17,51 +18,66 @@ import (
|
||||
// so one bucket's filer error doesn't kill the whole walk. Every
|
||||
// bucket's result goes to runStats when it is non-nil.
|
||||
//
|
||||
// A non-nil progress keeps an unfinished walk's finished buckets and
|
||||
// positions, and the next call for the same view and shards resumes
|
||||
// from them; a completed walk clears them, so the next one is full.
|
||||
// The returned time is when the earliest bucket the walk covers was
|
||||
// listed: its own start, or the start of the record it resumed.
|
||||
//
|
||||
// Composable shape: callers (the worker handler) supply a real
|
||||
// filer-backed ListFunc and a real Dispatcher (WalkerDispatcher).
|
||||
// Tests pass bootstrap.EntryCallback and a stub Dispatcher.
|
||||
func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher, runStats *RunStats) error {
|
||||
func WalkBuckets(ctx context.Context, view *engine.Snapshot, shards []int, buckets []string, list bootstrap.ListFunc, dispatch bootstrap.Dispatcher, runStats *RunStats, progress WalkCheckpointPersister) (time.Time, error) {
|
||||
if view == nil {
|
||||
return errors.New("WalkBuckets: nil view")
|
||||
return time.Time{}, errors.New("WalkBuckets: nil view")
|
||||
}
|
||||
if list == nil {
|
||||
return errors.New("WalkBuckets: nil list")
|
||||
return time.Time{}, errors.New("WalkBuckets: nil list")
|
||||
}
|
||||
if dispatch == nil {
|
||||
return errors.New("WalkBuckets: nil dispatch")
|
||||
return time.Time{}, errors.New("WalkBuckets: nil dispatch")
|
||||
}
|
||||
shardSet := make(map[int]bool, len(shards))
|
||||
for _, sh := range shards {
|
||||
shardSet[sh] = true
|
||||
}
|
||||
filtered := perShardListFunc(list, shardSet, runStats)
|
||||
walk := loadWalkProgress(ctx, progress, view, shards, buckets)
|
||||
var firstErr error
|
||||
for i, b := range buckets {
|
||||
if err := ctx.Err(); err != nil {
|
||||
for _, rest := range buckets[i:] {
|
||||
runStats.finishBucket(rest, bucketSkipped, nil)
|
||||
}
|
||||
return err
|
||||
_ = walk.finish(ctx, err)
|
||||
return walk.cp.StartedAt, err
|
||||
}
|
||||
if walk.done[b] {
|
||||
runStats.finishBucket(b, bucketWalked, nil)
|
||||
continue
|
||||
}
|
||||
runStats.startBucket(b)
|
||||
_, err := bootstrap.Walk(ctx, view, b, filtered, dispatch, bootstrap.WalkOptions{})
|
||||
_, err := bootstrap.Walk(ctx, view, b, walk.track(b, filtered), dispatch, bootstrap.WalkOptions{Resume: walk.cp.Resume[b]})
|
||||
switch {
|
||||
case err == nil:
|
||||
runStats.finishBucket(b, bucketWalked, nil)
|
||||
walk.bucketDone(ctx, b)
|
||||
continue
|
||||
case ctx.Err() != nil:
|
||||
runStats.finishBucket(b, bucketSkipped, nil)
|
||||
default:
|
||||
runStats.finishBucket(b, bucketFailed, err)
|
||||
}
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("walk %s: %w", b, err)
|
||||
} else {
|
||||
glog.V(1).Infof("walker: additional bucket %s: %v", b, err)
|
||||
}
|
||||
if firstErr == nil {
|
||||
firstErr = fmt.Errorf("walk %s: %w", b, err)
|
||||
} else {
|
||||
glog.V(1).Infof("walker: additional bucket %s: %v", b, err)
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
if err := walk.finish(ctx, firstErr); err != nil {
|
||||
return walk.cp.StartedAt, err
|
||||
}
|
||||
return walk.cp.StartedAt, firstErr
|
||||
}
|
||||
|
||||
// perShardListFunc wraps an inner ListFunc so only entries whose
|
||||
|
||||
@@ -104,8 +104,8 @@ func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) {
|
||||
|
||||
snap := snapshotForBucketRule(t, bucket, 7) // 30-day-old objects expire
|
||||
d := &recordingDispatcher{}
|
||||
err := WalkBuckets(context.Background(), snap, []int{targetShard}, []string{bucket},
|
||||
bootstrap.EntryCallback(all), d, nil)
|
||||
_, err := WalkBuckets(context.Background(), snap, []int{targetShard}, []string{bucket},
|
||||
bootstrap.EntryCallback(all), d, nil, nil)
|
||||
require.NoError(t, err)
|
||||
|
||||
gotPaths := make([]string, 0, len(d.calls))
|
||||
@@ -124,9 +124,12 @@ func TestWalkBuckets_DispatchesOnlyShardMatchingEntries(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWalkBuckets_NilGuards(t *testing.T) {
|
||||
require.Error(t, WalkBuckets(context.Background(), nil, []int{0}, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil))
|
||||
require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, nil, &recordingDispatcher{}, nil))
|
||||
require.Error(t, WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, bootstrap.EntryCallback(nil), nil, nil))
|
||||
_, err := WalkBuckets(context.Background(), nil, []int{0}, nil, bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil, nil)
|
||||
require.Error(t, err)
|
||||
_, err = WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, nil, &recordingDispatcher{}, nil, nil)
|
||||
require.Error(t, err)
|
||||
_, err = WalkBuckets(context.Background(), snapshotForBucketRule(t, "b", 7), []int{0}, nil, bootstrap.EntryCallback(nil), nil, nil, nil)
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) {
|
||||
@@ -141,7 +144,7 @@ func TestWalkBuckets_OneBucketErrorDoesNotStopOthers(t *testing.T) {
|
||||
}
|
||||
|
||||
snap := snapshotForBucketRule(t, "good", 7)
|
||||
err := WalkBuckets(context.Background(), snap, []int{0}, []string{"bad", "good"}, list, &recordingDispatcher{}, nil)
|
||||
_, err := WalkBuckets(context.Background(), snap, []int{0}, []string{"bad", "good"}, list, &recordingDispatcher{}, nil, nil)
|
||||
require.Error(t, err)
|
||||
assert.ErrorIs(t, err, listErr)
|
||||
}
|
||||
@@ -152,8 +155,8 @@ func TestWalkBuckets_HonorsContextCancellationBetweenBuckets(t *testing.T) {
|
||||
// short-circuit.
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), []int{0}, []string{"b"},
|
||||
bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil)
|
||||
_, err := WalkBuckets(ctx, snapshotForBucketRule(t, "b", 7), []int{0}, []string{"b"},
|
||||
bootstrap.EntryCallback(nil), &recordingDispatcher{}, nil, nil)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
)
|
||||
|
||||
// WalkCheckpoint is the saved progress of an unfinished WalkBuckets
|
||||
// call. ViewHash (engine.WalkContentHash) and Shards identify the walk;
|
||||
// a walk with another identity starts over.
|
||||
type WalkCheckpoint struct {
|
||||
ViewHash [32]byte
|
||||
Shards []int
|
||||
// Done lists finished buckets; Resume maps a partly walked bucket
|
||||
// to its bootstrap.Checkpoint.LastScannedPath.
|
||||
Done []string
|
||||
Resume map[string]string
|
||||
// When the walk that began this record started; every bucket in it
|
||||
// was listed after that.
|
||||
StartedAt time.Time
|
||||
// Set by LoadWalk; SaveWalk stamps the current time.
|
||||
SavedAt time.Time
|
||||
}
|
||||
|
||||
// WalkCheckpointPersister keeps one WalkCheckpoint per shard set.
|
||||
type WalkCheckpointPersister interface {
|
||||
LoadWalk(ctx context.Context, shards []int) (WalkCheckpoint, bool, error)
|
||||
SaveWalk(ctx context.Context, cp WalkCheckpoint) error
|
||||
}
|
||||
|
||||
type walkCheckpointFile struct {
|
||||
Version int `json:"version"`
|
||||
Shards []int `json:"shards"`
|
||||
ViewHash []byte `json:"view_hash,omitempty"`
|
||||
Done []string `json:"done,omitempty"`
|
||||
Resume map[string]string `json:"resume,omitempty"`
|
||||
StartedNs int64 `json:"started_ns,omitempty"`
|
||||
SavedNs int64 `json:"saved_ns,omitempty"`
|
||||
}
|
||||
|
||||
func walkCheckpointFileName(shards []int) string {
|
||||
contiguous := len(shards) > 1
|
||||
for i := 1; i < len(shards); i++ {
|
||||
contiguous = contiguous && shards[i] == shards[i-1]+1
|
||||
}
|
||||
if contiguous {
|
||||
return fmt.Sprintf("walk-%02d-%02d.json", shards[0], shards[len(shards)-1])
|
||||
}
|
||||
parts := make([]string, len(shards))
|
||||
for i, sh := range shards {
|
||||
parts[i] = fmt.Sprintf("%02d", sh)
|
||||
}
|
||||
return "walk-" + strings.Join(parts, "_") + ".json"
|
||||
}
|
||||
|
||||
func (p *FilerCursorPersister) LoadWalk(ctx context.Context, shards []int) (WalkCheckpoint, bool, error) {
|
||||
if p.Store == nil {
|
||||
return WalkCheckpoint{}, false, errors.New("FilerCursorPersister: nil Store")
|
||||
}
|
||||
content, err := p.Store.Read(ctx, CursorDir, walkCheckpointFileName(shards))
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
return WalkCheckpoint{}, false, nil
|
||||
}
|
||||
return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint read: %w", err)
|
||||
}
|
||||
var f walkCheckpointFile
|
||||
if err := json.Unmarshal(content, &f); err != nil {
|
||||
return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint decode: %w", err)
|
||||
}
|
||||
if f.Version != cursorFileVersion {
|
||||
return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint version: got %d, want %d", f.Version, cursorFileVersion)
|
||||
}
|
||||
if len(f.ViewHash) != 0 && len(f.ViewHash) != 32 {
|
||||
return WalkCheckpoint{}, false, fmt.Errorf("walk checkpoint view_hash: got %d bytes, want 32", len(f.ViewHash))
|
||||
}
|
||||
cp := WalkCheckpoint{Shards: f.Shards, Done: f.Done, Resume: f.Resume, SavedAt: time.Unix(0, f.SavedNs)}
|
||||
if f.StartedNs != 0 {
|
||||
cp.StartedAt = time.Unix(0, f.StartedNs)
|
||||
}
|
||||
copy(cp.ViewHash[:], f.ViewHash)
|
||||
return cp, true, nil
|
||||
}
|
||||
|
||||
func (p *FilerCursorPersister) SaveWalk(ctx context.Context, cp WalkCheckpoint) error {
|
||||
if p.Store == nil {
|
||||
return errors.New("FilerCursorPersister: nil Store")
|
||||
}
|
||||
f := walkCheckpointFile{Version: cursorFileVersion, Shards: cp.Shards, Done: cp.Done, Resume: cp.Resume, SavedNs: time.Now().UnixNano()}
|
||||
if !cp.StartedAt.IsZero() {
|
||||
f.StartedNs = cp.StartedAt.UnixNano()
|
||||
}
|
||||
if cp.ViewHash != ([32]byte{}) {
|
||||
f.ViewHash = cp.ViewHash[:]
|
||||
}
|
||||
content, err := json.Marshal(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("walk checkpoint encode: %w", err)
|
||||
}
|
||||
if err := p.Store.Save(ctx, CursorDir, walkCheckpointFileName(cp.Shards), content); err != nil {
|
||||
return fmt.Errorf("walk checkpoint save: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// walkCheckpointInterval spaces the saves of a running walk, in ns.
|
||||
var walkCheckpointInterval atomic.Int64
|
||||
|
||||
// A walk that keeps failing saves on every attempt; a record this old
|
||||
// belongs to a shard set that stopped walking, and resuming it would
|
||||
// skip buckets walked long ago.
|
||||
const walkCheckpointMaxAge = 7 * 24 * time.Hour
|
||||
|
||||
// A resumed walk that keeps failing without progress is treated as stuck
|
||||
// once its record has not advanced for this long.
|
||||
const walkCheckpointStallLimit = 24 * time.Hour
|
||||
|
||||
func init() { walkCheckpointInterval.Store(int64(30 * time.Second)) }
|
||||
|
||||
// walkProgress tracks one WalkBuckets call against its checkpoint.
|
||||
type walkProgress struct {
|
||||
store WalkCheckpointPersister
|
||||
cp WalkCheckpoint
|
||||
done map[string]bool
|
||||
onDisk bool // a saved record a finished walk must clear
|
||||
resumed bool
|
||||
progressed bool
|
||||
savedAt time.Time
|
||||
}
|
||||
|
||||
func loadWalkProgress(ctx context.Context, store WalkCheckpointPersister, view *engine.Snapshot, shards []int, buckets []string) *walkProgress {
|
||||
sorted := slices.Clone(shards)
|
||||
slices.Sort(sorted)
|
||||
now := time.Now()
|
||||
w := &walkProgress{
|
||||
store: store,
|
||||
cp: WalkCheckpoint{ViewHash: engine.WalkContentHash(view), Shards: sorted, Resume: map[string]string{}, StartedAt: now},
|
||||
done: map[string]bool{},
|
||||
savedAt: now,
|
||||
}
|
||||
if store == nil {
|
||||
return w
|
||||
}
|
||||
saved, found, err := store.LoadWalk(ctx, sorted)
|
||||
if err != nil {
|
||||
glog.Warningf("walker: shards %v: %v; walking every bucket", sorted, err)
|
||||
w.onDisk = true
|
||||
return w
|
||||
}
|
||||
if !found {
|
||||
return w
|
||||
}
|
||||
w.onDisk = len(saved.Done) > 0 || len(saved.Resume) > 0
|
||||
if saved.ViewHash != w.cp.ViewHash || !slices.Equal(saved.Shards, sorted) || saved.StartedAt.IsZero() || time.Since(saved.SavedAt) > walkCheckpointMaxAge {
|
||||
return w
|
||||
}
|
||||
listed := make(map[string]bool, len(buckets))
|
||||
for _, b := range buckets {
|
||||
listed[b] = true
|
||||
}
|
||||
for _, b := range saved.Done {
|
||||
if listed[b] {
|
||||
w.done[b] = true
|
||||
}
|
||||
}
|
||||
for b, pos := range saved.Resume {
|
||||
if listed[b] && !w.done[b] {
|
||||
w.cp.Resume[b] = pos
|
||||
}
|
||||
}
|
||||
w.cp.SavedAt = saved.SavedAt
|
||||
w.resumed = len(w.done) > 0 || len(w.cp.Resume) > 0
|
||||
if w.resumed {
|
||||
w.cp.StartedAt = saved.StartedAt
|
||||
glog.V(0).Infof("walker: shards %v resume with %d bucket(s) done and %d partly walked", sorted, len(w.done), len(w.cp.Resume))
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// track records each finished entry's position as the bucket's resume
|
||||
// point, saving it every walkCheckpointInterval.
|
||||
func (w *walkProgress) track(bucket string, list bootstrap.ListFunc) bootstrap.ListFunc {
|
||||
if w.store == nil {
|
||||
return list
|
||||
}
|
||||
return func(ctx context.Context, b, start string, cb func(*bootstrap.Entry) error) error {
|
||||
return list(ctx, b, start, func(e *bootstrap.Entry) error {
|
||||
if err := cb(e); err != nil {
|
||||
return err
|
||||
}
|
||||
if e != nil && e.Resume != "" {
|
||||
w.cp.Resume[bucket] = e.Resume
|
||||
w.progressed = true
|
||||
w.maybeSave(ctx)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (w *walkProgress) bucketDone(ctx context.Context, bucket string) {
|
||||
w.done[bucket] = true
|
||||
delete(w.cp.Resume, bucket)
|
||||
w.progressed = true
|
||||
w.maybeSave(ctx)
|
||||
}
|
||||
|
||||
func (w *walkProgress) maybeSave(ctx context.Context) {
|
||||
if w.store == nil || time.Since(w.savedAt) < time.Duration(walkCheckpointInterval.Load()) {
|
||||
return
|
||||
}
|
||||
w.savedAt = time.Now()
|
||||
if err := w.save(ctx); err != nil {
|
||||
glog.Warningf("walker: shards %v: %v", w.cp.Shards, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (w *walkProgress) save(ctx context.Context) error {
|
||||
cp := w.cp
|
||||
cp.Done = make([]string, 0, len(w.done))
|
||||
for b := range w.done {
|
||||
cp.Done = append(cp.Done, b)
|
||||
}
|
||||
slices.Sort(cp.Done)
|
||||
kept := len(cp.Done) > 0 || len(cp.Resume) > 0
|
||||
// A save that reports an error may still have been written.
|
||||
w.onDisk = w.onDisk || kept
|
||||
if err := w.store.SaveWalk(ctx, cp); err != nil {
|
||||
return err
|
||||
}
|
||||
w.onDisk = kept
|
||||
return nil
|
||||
}
|
||||
|
||||
// finish saves where the walk stopped, or clears the checkpoint when it
|
||||
// completed. A resumed walk that failed without progress on a record
|
||||
// older than walkCheckpointStallLimit is cleared too, so one stuck
|
||||
// bucket cannot keep the finished ones from ever being walked again.
|
||||
// Any other failed walk without progress has nothing new to record and
|
||||
// leaves the file, read or not, alone.
|
||||
func (w *walkProgress) finish(ctx context.Context, walkErr error) error {
|
||||
if w.store == nil {
|
||||
return nil
|
||||
}
|
||||
reset := walkErr == nil
|
||||
if walkErr != nil && !w.progressed {
|
||||
if ctx.Err() != nil || !w.resumed || time.Since(w.cp.SavedAt) < walkCheckpointStallLimit {
|
||||
return nil
|
||||
}
|
||||
glog.Warningf("walker: shards %v made no progress since %v; the next walk starts over", w.cp.Shards, w.cp.SavedAt.Format(time.RFC3339))
|
||||
reset = true
|
||||
}
|
||||
if reset {
|
||||
clear(w.done)
|
||||
clear(w.cp.Resume)
|
||||
}
|
||||
if !w.onDisk && len(w.done) == 0 && len(w.cp.Resume) == 0 {
|
||||
return nil
|
||||
}
|
||||
saveCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := w.save(saveCtx); err != nil {
|
||||
if walkErr == nil {
|
||||
// Left in place, the record would skip buckets next walk.
|
||||
return fmt.Errorf("walker: shards %v: %w", w.cp.Shards, err)
|
||||
}
|
||||
glog.Warningf("walker: shards %v: %v", w.cp.Shards, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,355 @@
|
||||
package dailyrun
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type scriptedDispatcher struct {
|
||||
calls []string
|
||||
fail map[string]bool
|
||||
}
|
||||
|
||||
func (d *scriptedDispatcher) Delete(_ context.Context, a *engine.CompiledAction, e *bootstrap.Entry) error {
|
||||
k := a.Bucket + "/" + e.Path + "@" + e.VersionID
|
||||
d.calls = append(d.calls, k)
|
||||
if d.fail[k] {
|
||||
return errors.New("delete failed")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type savingStore struct {
|
||||
*fakeStore
|
||||
saves [][]byte
|
||||
attempts int
|
||||
failAll bool
|
||||
lostReply bool
|
||||
}
|
||||
|
||||
func (s *savingStore) Read(ctx context.Context, dir, name string) ([]byte, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.fakeStore.Read(ctx, dir, name)
|
||||
}
|
||||
|
||||
func (s *savingStore) Save(ctx context.Context, dir, name string, content []byte) error {
|
||||
s.attempts++
|
||||
if s.failAll {
|
||||
return errors.New("filer down")
|
||||
}
|
||||
s.saves = append(s.saves, content)
|
||||
if err := s.fakeStore.Save(ctx, dir, name, content); err != nil {
|
||||
return err
|
||||
}
|
||||
if s.lostReply {
|
||||
return errors.New("response lost")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkpointTestView(t *testing.T, prefix string) *engine.Snapshot {
|
||||
t.Helper()
|
||||
rules := func() []*s3lifecycle.Rule {
|
||||
return []*s3lifecycle.Rule{{ID: "r", Status: s3lifecycle.StatusEnabled, Prefix: prefix, ExpirationDays: 1, NoncurrentVersionExpirationDays: 1}}
|
||||
}
|
||||
e := engine.New()
|
||||
e.Compile([]engine.CompileInput{
|
||||
{Bucket: "a", Rules: rules()},
|
||||
{Bucket: "b", Versioned: true, Rules: rules()},
|
||||
{Bucket: "c", Rules: rules()},
|
||||
}, engine.CompileOptions{})
|
||||
snap := e.Snapshot()
|
||||
for _, a := range snap.AllActions() {
|
||||
snap.MarkActive(a.Key)
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
func checkpointTestFiler() *fakeFiler {
|
||||
t1 := time.Now().Add(-30 * 24 * time.Hour)
|
||||
t2, t3 := t1.Add(time.Hour), t1.Add(2*time.Hour)
|
||||
ver := func(id string, mtime time.Time) *filer_pb.Entry {
|
||||
return fileWithExt(id, mtime, 1, map[string][]byte{s3_constants.ExtVersionIdKey: []byte(id)})
|
||||
}
|
||||
vs := s3_constants.VersionsFolder
|
||||
return &fakeFiler{tree: map[string][]*filer_pb.Entry{
|
||||
"/buckets/a": {file("a1", t1, 1), file("a2", t1, 1)},
|
||||
"/buckets/b": {file("b0", t1, 1), versionsDir("k1"+vs, ""), versionsDir("obj"+vs, ""), file("z", t1, 1)},
|
||||
"/buckets/b/k1" + vs: {ver("k1a", t1), ver("k1b", t2)},
|
||||
"/buckets/b/obj" + vs: {ver("v1", t1), ver("v2", t2), ver("v3", t3)},
|
||||
"/buckets/c": {file("c1", t1, 1)},
|
||||
}}
|
||||
}
|
||||
|
||||
var fullWalk = []string{
|
||||
"a/a1@", "a/a2@",
|
||||
"b/k1@k1b", "b/k1@k1a", "b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@",
|
||||
"c/c1@",
|
||||
}
|
||||
|
||||
type checkpointHarness struct {
|
||||
t *testing.T
|
||||
client *fakeFiler
|
||||
store *savingStore
|
||||
p *FilerCursorPersister
|
||||
stats *RunStats
|
||||
}
|
||||
|
||||
func newCheckpointHarness(t *testing.T) *checkpointHarness {
|
||||
store := &savingStore{fakeStore: newFakeStore()}
|
||||
return &checkpointHarness{t: t, client: checkpointTestFiler(), store: store, p: &FilerCursorPersister{Store: store}}
|
||||
}
|
||||
|
||||
func (h *checkpointHarness) walk(ctx context.Context, view *engine.Snapshot, shards []int, fail ...string) ([]string, error) {
|
||||
d := &scriptedDispatcher{fail: map[string]bool{}}
|
||||
for _, f := range fail {
|
||||
d.fail[f] = true
|
||||
}
|
||||
_, err := WalkBuckets(ctx, view, shards, []string{"a", "b", "c"}, FilerListFunc(h.client, "/buckets", view), d, h.stats, h.p)
|
||||
return d.calls, err
|
||||
}
|
||||
|
||||
func (h *checkpointHarness) saved(shards []int) WalkCheckpoint {
|
||||
cp, found, err := h.p.LoadWalk(context.Background(), shards)
|
||||
require.NoError(h.t, err)
|
||||
require.True(h.t, found)
|
||||
return cp
|
||||
}
|
||||
|
||||
func TestWalkBuckets_ResumedWalkCountsEarlierBucketsAsWalked(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
_, err := h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
|
||||
h.stats = &RunStats{}
|
||||
_, err = h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
s := h.stats.Summary()
|
||||
assert.Equal(t, 3, s.BucketsWalked)
|
||||
assert.Zero(t, s.BucketsFailed)
|
||||
assert.Equal(t, int64(5), s.EntriesVisited, "only the rest of b is visited")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_ResumesAfterFailureInsideVersionGroup(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
|
||||
calls, err := h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, append(fullWalk[:7:7], "c/c1@"), calls, "a failed bucket does not stop the others")
|
||||
cp := h.saved(allShards())
|
||||
assert.Equal(t, []string{"a", "c"}, cp.Done)
|
||||
assert.Equal(t, map[string]string{"b": "1:k1.versions"}, cp.Resume, "the position stays before the unfinished group")
|
||||
assert.Equal(t, engine.WalkContentHash(view), cp.ViewHash)
|
||||
|
||||
calls, err = h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@"}, calls)
|
||||
cp = h.saved(allShards())
|
||||
assert.Empty(t, cp.Done)
|
||||
assert.Empty(t, cp.Resume)
|
||||
|
||||
calls, err = h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fullWalk, calls, "after a completed walk the next one is full")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_ResumedWalkReportsTheRecordStart(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
walk := func(fail ...string) (time.Time, error) {
|
||||
d := &scriptedDispatcher{fail: map[string]bool{}}
|
||||
for _, f := range fail {
|
||||
d.fail[f] = true
|
||||
}
|
||||
return WalkBuckets(context.Background(), view, allShards(), []string{"a", "b", "c"}, FilerListFunc(h.client, "/buckets", view), d, nil, h.p)
|
||||
}
|
||||
|
||||
before := time.Now()
|
||||
since, err := walk("b/obj@v1")
|
||||
require.Error(t, err)
|
||||
assert.False(t, since.Before(before))
|
||||
assert.Equal(t, since.UnixNano(), h.saved(allShards()).StartedAt.UnixNano())
|
||||
|
||||
name := CursorDir + "/walk-00-15.json"
|
||||
var f walkCheckpointFile
|
||||
require.NoError(t, json.Unmarshal(h.store.files[name], &f))
|
||||
started := time.Now().Add(-48 * time.Hour)
|
||||
f.StartedNs = started.UnixNano()
|
||||
h.store.files[name], _ = json.Marshal(f)
|
||||
since, err = walk()
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, started.UnixNano(), since.UnixNano(), "a resumed walk dates from the walk that listed its earlier buckets")
|
||||
|
||||
before = time.Now()
|
||||
since, err = walk()
|
||||
require.NoError(t, err)
|
||||
assert.False(t, since.Before(before), "a full walk dates from its own start")
|
||||
|
||||
_, err = walk("b/obj@v1")
|
||||
require.Error(t, err)
|
||||
require.NoError(t, json.Unmarshal(h.store.files[name], &f))
|
||||
f.StartedNs = 0
|
||||
h.store.files[name], _ = json.Marshal(f)
|
||||
before = time.Now()
|
||||
since, err = walk()
|
||||
require.NoError(t, err)
|
||||
assert.False(t, since.Before(before), "a record without a start is not resumed")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_CheckpointBoundToViewAndShards(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
_, err := h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
|
||||
calls, err := h.walk(context.Background(), checkpointTestView(t, "o"), allShards())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1"}, calls, "another rule set walks from the start")
|
||||
|
||||
_, err = h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
var some []int
|
||||
for _, sh := range allShards() {
|
||||
if sh != 3 {
|
||||
some = append(some, sh)
|
||||
}
|
||||
}
|
||||
calls, err = h.walk(context.Background(), view, some)
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, calls, "b/k1@k1b", "another shard set walks from the start")
|
||||
assert.Equal(t, map[string]string{"b": "1:k1.versions"}, h.saved(allShards()).Resume, "and keeps its own record")
|
||||
assert.Contains(t, h.store.files, CursorDir+"/walk-00-15.json")
|
||||
|
||||
var f walkCheckpointFile
|
||||
require.NoError(t, json.Unmarshal(h.store.files[CursorDir+"/walk-00-15.json"], &f))
|
||||
f.SavedNs = time.Now().Add(-walkCheckpointMaxAge - time.Hour).UnixNano()
|
||||
h.store.files[CursorDir+"/walk-00-15.json"], _ = json.Marshal(f)
|
||||
calls, err = h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, fullWalk, calls, "a record nobody saved for too long is not resumed")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_StuckResumeStartsOver(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
_, err := h.walk(context.Background(), view, allShards(), "b/obj@v2")
|
||||
require.Error(t, err)
|
||||
before := h.saved(allShards())
|
||||
|
||||
calls, err := h.walk(context.Background(), view, allShards(), "b/obj@v2")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, []string{"b/obj@v3", "b/obj@v2"}, calls)
|
||||
assert.Equal(t, before, h.saved(allShards()), "a recent record survives a resume without progress")
|
||||
|
||||
var f walkCheckpointFile
|
||||
require.NoError(t, json.Unmarshal(h.store.files[CursorDir+"/walk-00-15.json"], &f))
|
||||
f.SavedNs = time.Now().Add(-walkCheckpointStallLimit - time.Hour).UnixNano()
|
||||
h.store.files[CursorDir+"/walk-00-15.json"], _ = json.Marshal(f)
|
||||
calls, err = h.walk(context.Background(), view, allShards(), "b/obj@v2")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, []string{"b/obj@v3", "b/obj@v2"}, calls)
|
||||
assert.Empty(t, h.saved(allShards()).Done, "a resume stuck for too long is dropped")
|
||||
|
||||
calls, err = h.walk(context.Background(), view, allShards(), "b/obj@v2")
|
||||
require.Error(t, err)
|
||||
assert.Equal(t, fullWalk[:6], calls[:6], "so the finished buckets get walked again")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_CanceledResumeKeepsCheckpoint(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
_, err := h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
before := h.saved(allShards())
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = h.walk(ctx, view, allShards())
|
||||
require.ErrorIs(t, err, context.Canceled)
|
||||
assert.Equal(t, before, h.saved(allShards()), "a walk that could not read the record leaves it alone")
|
||||
}
|
||||
|
||||
func TestWalkBuckets_ClearFailureFailsTheWalk(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
_, err := h.walk(context.Background(), view, allShards(), "b/obj@v1")
|
||||
require.Error(t, err)
|
||||
|
||||
h.store.failAll = true
|
||||
_, err = h.walk(context.Background(), view, allShards())
|
||||
require.Error(t, err, "a record left behind would skip buckets on the next walk")
|
||||
|
||||
h.store.failAll = false
|
||||
calls, err := h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"b/obj@v3", "b/obj@v2", "b/obj@v1", "b/b0@", "b/z@"}, calls)
|
||||
}
|
||||
|
||||
func TestWalkBuckets_ResumesFromPeriodicSave(t *testing.T) {
|
||||
prev := walkCheckpointInterval.Load()
|
||||
walkCheckpointInterval.Store(0)
|
||||
defer walkCheckpointInterval.Store(prev)
|
||||
|
||||
h := newCheckpointHarness(t)
|
||||
view := checkpointTestView(t, "")
|
||||
calls, err := h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, fullWalk, calls)
|
||||
|
||||
// Replay a crash right after each periodic save.
|
||||
saves := h.store.saves
|
||||
require.Greater(t, len(saves), 5)
|
||||
left := len(fullWalk)
|
||||
for _, content := range saves[:len(saves)-1] {
|
||||
var f walkCheckpointFile
|
||||
require.NoError(t, json.Unmarshal(content, &f))
|
||||
h.store.files[CursorDir+"/walk-00-15.json"] = content
|
||||
calls, err := h.walk(context.Background(), view, allShards())
|
||||
require.NoError(t, err)
|
||||
require.LessOrEqual(t, len(calls), left, "resume after %s", content)
|
||||
left = len(calls)
|
||||
assert.Equal(t, append([]string(nil), fullWalk[len(fullWalk)-left:]...), calls, "resume after %s", content)
|
||||
}
|
||||
assert.Zero(t, left)
|
||||
}
|
||||
|
||||
func TestWalkProgress_FailedPeriodicSaveWaitsForTheInterval(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
h.store.failAll = true
|
||||
w := loadWalkProgress(context.Background(), h.p, checkpointTestView(t, ""), allShards(), []string{"a"})
|
||||
w.savedAt = time.Now().Add(-time.Hour)
|
||||
w.bucketDone(context.Background(), "a")
|
||||
w.bucketDone(context.Background(), "a")
|
||||
assert.Equal(t, 1, h.store.attempts, "a failed save is retried a full interval later, not on the next entry")
|
||||
}
|
||||
|
||||
func TestWalkProgress_SaveWithLostReplyIsClearedOnCompletion(t *testing.T) {
|
||||
h := newCheckpointHarness(t)
|
||||
h.store.lostReply = true
|
||||
w := loadWalkProgress(context.Background(), h.p, checkpointTestView(t, ""), allShards(), []string{"a", "b"})
|
||||
w.savedAt = time.Now().Add(-time.Hour)
|
||||
w.bucketDone(context.Background(), "a")
|
||||
require.Equal(t, []string{"a"}, h.saved(allShards()).Done)
|
||||
|
||||
h.store.lostReply = false
|
||||
w.bucketDone(context.Background(), "b")
|
||||
require.NoError(t, w.finish(context.Background(), nil))
|
||||
cp := h.saved(allShards())
|
||||
assert.Empty(t, cp.Done, "the completed walk clears the record its failed save wrote")
|
||||
assert.Empty(t, cp.Resume)
|
||||
}
|
||||
@@ -27,6 +27,7 @@ func loadShardCursors(ctx context.Context, cfg Config) map[int]cursorLoad {
|
||||
// runShard goroutines wait on done instead of running their own listing.
|
||||
type pendingWalk struct {
|
||||
done chan struct{}
|
||||
since time.Time
|
||||
err error
|
||||
recovery bool
|
||||
}
|
||||
@@ -76,7 +77,7 @@ func planWalks(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow ti
|
||||
}
|
||||
go func() {
|
||||
defer close(pw.done)
|
||||
pw.err = cfg.Walker(ctx, view, shards)
|
||||
pw.since, pw.err = cfg.Walker(ctx, view, shards)
|
||||
}()
|
||||
}
|
||||
if len(recovery) > 0 {
|
||||
@@ -92,13 +93,13 @@ func planWalks(ctx context.Context, cfg Config, snap *engine.Snapshot, runNow ti
|
||||
// this branch (same recovery/steady kind), or runs a per-shard walk when
|
||||
// the plan missed — a cursor drift between the plan load and runShard's
|
||||
// own load — or no plan exists (direct runShard calls, tests).
|
||||
func walkForShard(ctx context.Context, walker WalkerFunc, walks map[int]*pendingWalk, shardID int, recovery bool, view *engine.Snapshot) error {
|
||||
func walkForShard(ctx context.Context, walker WalkerFunc, walks map[int]*pendingWalk, shardID int, recovery bool, view *engine.Snapshot) (time.Time, error) {
|
||||
if pw := walks[shardID]; pw != nil && pw.recovery == recovery {
|
||||
select {
|
||||
case <-pw.done:
|
||||
return pw.err
|
||||
return pw.since, pw.err
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
return time.Time{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return walker(ctx, view, []int{shardID})
|
||||
|
||||
@@ -44,11 +44,11 @@ func TestRun_SharesBucketWalkAcrossShards(t *testing.T) {
|
||||
Client: stubLifecycleClient{},
|
||||
Persister: p,
|
||||
Lister: stubSiblingLister{},
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, shards []int) error {
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
calls = append(calls, append([]int{}, shards...))
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
}
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
@@ -118,12 +118,12 @@ func TestRun_RecoveryAndSteadyGroupsWalkSeparately(t *testing.T) {
|
||||
Client: stubLifecycleClient{},
|
||||
Persister: p,
|
||||
Lister: stubSiblingLister{},
|
||||
Walker: func(_ context.Context, view *engine.Snapshot, shards []int) error {
|
||||
Walker: func(_ context.Context, view *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
calls = append(calls, append([]int{}, shards...))
|
||||
actions = append(actions, len(view.AllActions()))
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
}
|
||||
require.NoError(t, Run(context.Background(), cfg))
|
||||
@@ -161,8 +161,8 @@ func TestRun_SharedWalkErrorFailsCoveredShards(t *testing.T) {
|
||||
Client: stubLifecycleClient{},
|
||||
Persister: newMemPersister(),
|
||||
Lister: stubSiblingLister{},
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error {
|
||||
return errors.New("walk boom")
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) {
|
||||
return time.Time{}, errors.New("walk boom")
|
||||
},
|
||||
}
|
||||
err := Run(context.Background(), cfg)
|
||||
@@ -212,10 +212,10 @@ func TestRun_JoinsPlannedWalkWhenShardExitsEarly(t *testing.T) {
|
||||
Client: stubLifecycleClient{},
|
||||
Persister: &countingPersister{inner: newMemPersister(), failAfter: 1},
|
||||
Lister: stubSiblingLister{},
|
||||
Walker: func(context.Context, *engine.Snapshot, []int) error {
|
||||
Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) {
|
||||
close(walkStarted)
|
||||
<-walkRelease
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
}
|
||||
runDone := make(chan error, 1)
|
||||
|
||||
@@ -8,7 +8,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/bootstrap"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3lifecycle/engine"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
@@ -24,6 +23,9 @@ type WalkerDispatcher struct {
|
||||
// daily-run's processMatches uses so the walker and replay paths
|
||||
// can't combine to burst past the cap. nil disables throttling.
|
||||
Limiter *rate.Limiter
|
||||
// DeleteTimeout bounds each LifecycleDelete, applied after the
|
||||
// limiter wait. <= 0 uses DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
// nil skips per-pass outcome counts.
|
||||
Stats *RunStats
|
||||
}
|
||||
@@ -66,27 +68,13 @@ func (d *WalkerDispatcher) Delete(ctx context.Context, action *engine.CompiledAc
|
||||
ExpectedIdentity: toProtoIdentity(entry.Identity),
|
||||
}
|
||||
kindLabel := action.Key.ActionKind.String()
|
||||
if d.Limiter != nil {
|
||||
waitStart := time.Now()
|
||||
if waitErr := d.Limiter.Wait(ctx); waitErr != nil {
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: limiter: %w", action.Bucket, objectPath, action.Key.ActionKind, waitErr)
|
||||
}
|
||||
stats.S3LifecycleDispatchLimiterWaitSeconds.Observe(time.Since(waitStart).Seconds())
|
||||
}
|
||||
resp, err := d.Client.LifecycleDelete(ctx, req)
|
||||
resp, err := deleteWithRetry(ctx, d.Client, d.Limiter, req, d.DeleteTimeout)
|
||||
if err != nil {
|
||||
// "RPC_ERROR" matches the streaming dispatcher and processMatches
|
||||
// labels so transport-class failures aggregate under one key.
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR")
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: %w", action.Bucket, objectPath, action.Key.ActionKind, err)
|
||||
}
|
||||
if resp == nil {
|
||||
// A misbehaving server stub returning (nil, nil) would panic on
|
||||
// the switch below. Bucket under RPC_ERROR rather than a new
|
||||
// label — operationally it's the same class of failure.
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, "RPC_ERROR")
|
||||
return fmt.Errorf("walker dispatch %s/%s %s: nil response", action.Bucket, objectPath, action.Key.ActionKind)
|
||||
}
|
||||
countDispatch(d.Stats, action.Bucket, kindLabel, resp.Outcome.String())
|
||||
switch resp.Outcome {
|
||||
case s3_lifecycle_pb.LifecycleDeleteOutcome_DONE,
|
||||
|
||||
@@ -84,9 +84,9 @@ func TestRunShard_WalkerThrottle(t *testing.T) {
|
||||
calls := 0
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error {
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) {
|
||||
calls++
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
RetentionWindow: -1, // negative sentinel: falls back to maxTTL — leaves walkView empty
|
||||
WalkerInterval: tc.interval,
|
||||
@@ -224,9 +224,9 @@ func TestRunShard_ColdStartDoesNotDoubleWalk(t *testing.T) {
|
||||
calls := 0
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error {
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) {
|
||||
calls++
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
WalkerInterval: 0,
|
||||
}
|
||||
@@ -259,9 +259,9 @@ func TestRunShard_RecoveryWalkerSetsLastWalkedAnchor(t *testing.T) {
|
||||
calls := 0
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error {
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) {
|
||||
calls++
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
WalkerInterval: time.Hour,
|
||||
}
|
||||
|
||||
@@ -70,11 +70,11 @@ func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) {
|
||||
calls := 0
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(_ context.Context, view *engine.Snapshot, shards []int) error {
|
||||
Walker: func(_ context.Context, view *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
calls++
|
||||
gotView = view
|
||||
gotShards = shards
|
||||
return nil
|
||||
return time.Time{}, nil
|
||||
},
|
||||
}
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
@@ -93,6 +93,68 @@ func TestRunShard_WalkerInvokedOnRecoveryBranch(t *testing.T) {
|
||||
assert.NotEqual(t, stale, got.RuleSetHash, "stale hash must be replaced")
|
||||
}
|
||||
|
||||
func TestRunShard_RecoveryReplaysFromResumedWalkStart(t *testing.T) {
|
||||
snap := snapshotWithRule(t, 30)
|
||||
p := newMemPersister()
|
||||
var stale [32]byte
|
||||
for i := range stale {
|
||||
stale[i] = 0xAD
|
||||
}
|
||||
require.NoError(t, p.Save(context.Background(), 3, Cursor{TsNs: 1234, RuleSetHash: stale}))
|
||||
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
walkedFrom := runNow.Add(-48 * time.Hour)
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) {
|
||||
return walkedFrom, nil
|
||||
},
|
||||
}
|
||||
require.NoError(t, runShard(context.Background(), cfg, snap, runNow, 3, nil, nil))
|
||||
|
||||
got, ok, err := p.Load(context.Background(), 3)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, walkedFrom.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs,
|
||||
"objects in buckets a resumed walk listed earlier must stay in the replay window")
|
||||
assert.Equal(t, runNow.UnixNano(), got.LastWalkedNs)
|
||||
}
|
||||
|
||||
func TestRunShard_ColdStartWithResumedWalkReplaysNextPass(t *testing.T) {
|
||||
snap := snapshotWithRule(t, 30)
|
||||
p := newMemPersister()
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
walkedFrom := runNow.Add(-48 * time.Hour)
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(context.Context, *engine.Snapshot, []int) (time.Time, error) {
|
||||
return walkedFrom, nil
|
||||
},
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
require.NoError(t, runShard(ctx, cfg, snap, runNow, 3, nil, nil))
|
||||
|
||||
got, ok, err := p.Load(context.Background(), 3)
|
||||
require.NoError(t, err)
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, walkedFrom.Add(-engine.MaxEffectiveTTL(snap)).UnixNano(), got.TsNs,
|
||||
"this pass's subscription began at runNow - maxTTL, so the next pass replays from the walk start")
|
||||
assert.Equal(t, engine.ReplayContentHash(snap), got.RuleSetHash)
|
||||
}
|
||||
|
||||
func TestComputeGlobalStartTsNs_SavedCursorBelowTheFloor(t *testing.T) {
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
maxTTL := 30 * 24 * time.Hour
|
||||
floor := runNow.Add(-maxTTL).UnixNano()
|
||||
saved := runNow.Add(-maxTTL - 48*time.Hour).UnixNano()
|
||||
cfg := Config{Shards: []int{1, 3}}
|
||||
cursors := map[int]cursorLoad{3: {cursor: Cursor{TsNs: saved}, found: true}}
|
||||
assert.Equal(t, saved, computeGlobalStartTsNs(cursors, cfg, runNow, maxTTL),
|
||||
"a cursor a resumed cold start saved below runNow - maxTTL opens the next subscription")
|
||||
assert.Equal(t, floor, computeGlobalStartTsNs(map[int]cursorLoad{}, cfg, runNow, maxTTL))
|
||||
}
|
||||
|
||||
func TestRunShard_NilWalkerOnRecoveryIsNoop(t *testing.T) {
|
||||
// Phase 4a behavior: a nil Walker must not crash and must still
|
||||
// rewind the cursor.
|
||||
@@ -128,8 +190,8 @@ func TestRunShard_WalkerErrorPropagates(t *testing.T) {
|
||||
|
||||
cfg := Config{
|
||||
Persister: p,
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) error {
|
||||
return errors.New("walker boom")
|
||||
Walker: func(_ context.Context, _ *engine.Snapshot, _ []int) (time.Time, error) {
|
||||
return time.Time{}, errors.New("walker boom")
|
||||
},
|
||||
}
|
||||
runNow := time.Unix(1_700_000_000, 0).UTC()
|
||||
|
||||
@@ -171,6 +171,40 @@ func PromotedHash(s *Snapshot, retentionWindow time.Duration) [32]byte {
|
||||
return w.sum()
|
||||
}
|
||||
|
||||
// WalkContentHash identifies the walk a view drives: every non-disabled
|
||||
// action and each of their buckets' versioning, which shapes the
|
||||
// listing. Empty for a nil or empty view.
|
||||
func WalkContentHash(s *Snapshot) [32]byte {
|
||||
var empty [32]byte
|
||||
if s == nil {
|
||||
return empty
|
||||
}
|
||||
var items []hashItem
|
||||
for k, a := range s.actions {
|
||||
if a == nil || a.Mode == ModeDisabled {
|
||||
continue
|
||||
}
|
||||
items = append(items, hashItem{key: k, action: a})
|
||||
}
|
||||
if len(items) == 0 {
|
||||
return empty
|
||||
}
|
||||
sortHashItems(items)
|
||||
|
||||
w := newHashWriter()
|
||||
for _, it := range items {
|
||||
w.writeField(0x01, []byte(it.key.Bucket))
|
||||
w.writeField(0x02, it.key.RuleHash[:])
|
||||
w.writeInt(0x03, int64(it.key.ActionKind))
|
||||
versioned := int64(0)
|
||||
if s.BucketVersioned(it.key.Bucket) {
|
||||
versioned = 1
|
||||
}
|
||||
w.writeInt(0x04, versioned)
|
||||
}
|
||||
return w.sum()
|
||||
}
|
||||
|
||||
// MaxEffectiveTTL returns the maximum effective TTL across the *active*
|
||||
// replay-eligible actions in s. Returns 0 for a nil snapshot or one with no
|
||||
// active replay actions; the caller is expected to be in the empty-replay
|
||||
|
||||
@@ -271,3 +271,26 @@ func TestMaxEffectiveTTL_IgnoresInactiveActions(t *testing.T) {
|
||||
assert.Equal(t, time.Duration(0), MaxEffectiveTTL(snap),
|
||||
"inactive replay actions must not contribute to MaxEffectiveTTL")
|
||||
}
|
||||
|
||||
func TestWalkContentHash_SeparatesWalksAndEdits(t *testing.T) {
|
||||
var empty [32]byte
|
||||
assert.Equal(t, empty, WalkContentHash(nil))
|
||||
|
||||
rules := []*s3lifecycle.Rule{
|
||||
{ID: "days", Status: s3lifecycle.StatusEnabled, ExpirationDays: 30},
|
||||
{ID: "date", Status: s3lifecycle.StatusEnabled, Prefix: "d/", ExpirationDate: time.Now().Add(24 * time.Hour)},
|
||||
}
|
||||
snap := buildSnapshotForViews(t, "b1", rules...)
|
||||
_, walk := snap.RulesForShard(0, s3lifecycle.DaysToDuration(365))
|
||||
require.NotNil(t, walk)
|
||||
recovery := RecoveryView(snap)
|
||||
assert.NotEqual(t, WalkContentHash(walk), WalkContentHash(recovery), "steady and recovery walks cover different rules")
|
||||
assert.Equal(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(buildSnapshotForViews(t, "b1", rules[1], rules[0]))))
|
||||
|
||||
edited := buildSnapshotForViews(t, "b1", rules[0], &s3lifecycle.Rule{ID: "date", Status: s3lifecycle.StatusEnabled, Prefix: "e/", ExpirationDate: rules[1].ExpirationDate})
|
||||
assert.NotEqual(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(edited)))
|
||||
|
||||
e := New()
|
||||
versioned := e.Compile([]CompileInput{{Bucket: "b1", Versioned: true, Rules: rules}}, CompileOptions{})
|
||||
assert.NotEqual(t, WalkContentHash(recovery), WalkContentHash(RecoveryView(versioned)), "versioning changes the listing")
|
||||
}
|
||||
@@ -245,6 +245,9 @@ func loadClientTLS(config *util.ViperProxy, component, targetComponent string) (
|
||||
},
|
||||
VerificationType: advancedtls.CertVerification,
|
||||
}
|
||||
if config.GetBool(component+".verify_server_host") || config.GetBool("grpc.verify_server_host") {
|
||||
options.VerificationType = advancedtls.CertAndHostVerification
|
||||
}
|
||||
targetPrefix := ""
|
||||
if targetComponent != component {
|
||||
targetPrefix = strings.TrimPrefix(targetComponent, "grpc.") + "_"
|
||||
@@ -263,6 +266,8 @@ func loadClientTLS(config *util.ViperProxy, component, targetComponent string) (
|
||||
}
|
||||
if len(auther.AllowedCommonNames) != 0 || auther.AllowedWildcardDomain != "" {
|
||||
options.AdditionalPeerVerification = auther.AuthenticateServer
|
||||
} else if key := unappliedTargetAllowlist(config, component, targetComponent); key != "" {
|
||||
return nil, fmt.Errorf("%s is set, but %s dials every server with one allowlist: set %s.allowed_server_commonNames", key, component, component)
|
||||
} else {
|
||||
options.AdditionalPeerVerification = func(params *advancedtls.HandshakeVerificationInfo) (*advancedtls.PostHandshakeVerificationResults, error) {
|
||||
return &advancedtls.PostHandshakeVerificationResults{}, nil
|
||||
@@ -346,6 +351,44 @@ func checkTLSFiles(certFile, keyFile, caFile string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// targetAllowlistClients are the components that dial some targets through
|
||||
// LoadClientTLSForTarget, so their <target>_allowed_server_* keys apply.
|
||||
var targetAllowlistClients = map[string]bool{"grpc.filer": true}
|
||||
|
||||
// unappliedTargetAllowlist returns a per-target server allowlist key set under
|
||||
// a component that only dials with one option, which would otherwise leave
|
||||
// that component accepting any CA-signed server.
|
||||
func unappliedTargetAllowlist(config *util.ViperProxy, component, targetComponent string) string {
|
||||
if targetComponent != component || targetAllowlistClients[component] {
|
||||
return ""
|
||||
}
|
||||
prefix := strings.ToLower(component) + "."
|
||||
candidates := config.AllKeys()
|
||||
envPrefix := "WEED_" + strings.ToUpper(strings.ReplaceAll(component, ".", "_")) + "_"
|
||||
for _, env := range os.Environ() {
|
||||
if name, _, ok := strings.Cut(env, "="); ok && strings.HasPrefix(name, envPrefix) {
|
||||
candidates = append(candidates, prefix+strings.ToLower(strings.TrimPrefix(name, envPrefix)))
|
||||
}
|
||||
}
|
||||
var keys []string
|
||||
for _, key := range candidates {
|
||||
name, ok := strings.CutPrefix(key, prefix)
|
||||
if !ok || strings.Contains(name, ".") || config.GetString(key) == "" {
|
||||
continue
|
||||
}
|
||||
for _, suffix := range []string{"_allowed_server_commonnames", "_allowed_server_wildcard_domain"} {
|
||||
if target, ok := strings.CutSuffix(name, suffix); ok && target != "" {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(keys) == 0 {
|
||||
return ""
|
||||
}
|
||||
slices.Sort(keys)
|
||||
return keys[0]
|
||||
}
|
||||
|
||||
func parseAllowedCommonNames(commonNames string) map[string]bool {
|
||||
allowed := make(map[string]bool)
|
||||
for _, s := range strings.Split(commonNames, ",") {
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -178,6 +179,42 @@ func TestLoadClientTLSFallsBackToServingCert(t *testing.T) {
|
||||
|
||||
// A serverAuth-only cert presented as the client identity fails the peer's
|
||||
// clientAuth EKU verification — the failure mode client_cert exists to fix.
|
||||
func TestLoadClientTLSVerifyServerHost(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca := newTestCA(t, dir)
|
||||
serverCert, serverKey := ca.issueWithSANs(t, dir, "seaweed-master", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, []string{"master.invalid"}, nil)
|
||||
clientCert, clientKey := ca.issue(t, dir, "client", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})
|
||||
|
||||
v := util.NewViperProxy(viper.New())
|
||||
v.Set("grpc.ca", ca.file)
|
||||
v.Set("grpc.master.cert", serverCert)
|
||||
v.Set("grpc.master.key", serverKey)
|
||||
v.Set("grpc.client.cert", clientCert)
|
||||
v.Set("grpc.client.key", clientKey)
|
||||
v.Set("grpc.client.allowed_server_commonNames", "seaweed-master")
|
||||
|
||||
addr := startTestGrpcServer(t, v, "grpc.master")
|
||||
_, port, _ := net.SplitHostPort(addr)
|
||||
byName := net.JoinHostPort("localhost", port)
|
||||
if err := healthCheck(t, byName, LoadClientTLS(v, "grpc.client")); err != nil {
|
||||
t.Fatalf("the dialed host is not checked by default: %v", err)
|
||||
}
|
||||
|
||||
v.Set("grpc.verify_server_host", true)
|
||||
if err := healthCheck(t, byName, LoadClientTLS(v, "grpc.client")); err == nil {
|
||||
t.Fatal("expected a certificate for master.invalid to be rejected on a connection to localhost")
|
||||
}
|
||||
if err := healthCheck(t, addr, LoadClientTLS(v, "grpc.client")); err != nil {
|
||||
t.Fatalf("the certificate's IP SAN matches the dialed address: %v", err)
|
||||
}
|
||||
|
||||
v.Set("grpc.verify_server_host", false)
|
||||
v.Set("grpc.client.verify_server_host", true)
|
||||
if err := healthCheck(t, byName, LoadClientTLSForTarget(v, "grpc.client", "grpc.master")); err == nil {
|
||||
t.Fatal("expected the component setting to check the dialed host")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadClientTLSServerOnlyEkuRejected(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca := newTestCA(t, dir)
|
||||
@@ -531,3 +568,76 @@ func TestLoadClientTLSForTargetFallsBackToClientAllowlist(t *testing.T) {
|
||||
t.Fatalf("health check with client component allowlist failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadClientTLSRejectsUnappliedTargetAllowlist(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca := newTestCA(t, dir)
|
||||
certFile, keyFile := ca.issue(t, dir, "client", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})
|
||||
|
||||
v := util.NewViperProxy(viper.New())
|
||||
v.Set("grpc.ca", ca.file)
|
||||
v.Set("grpc.client.cert", certFile)
|
||||
v.Set("grpc.client.key", keyFile)
|
||||
v.Set("grpc.client.filer_allowed_server_commonNames", "seaweed-filer")
|
||||
v.Set("grpc.client.master_allowed_server_wildcard_domain", ".master.example.com")
|
||||
|
||||
if _, err := loadClientTLS(v, "grpc.client", "grpc.client"); err == nil {
|
||||
t.Fatal("expected error when per-target allowlists are the only allowlist of a single-option client")
|
||||
}
|
||||
|
||||
v.Set("grpc.client.allowed_server_commonNames", "seaweed-filer,seaweed-master")
|
||||
if _, err := loadClientTLS(v, "grpc.client", "grpc.client"); err != nil {
|
||||
t.Fatalf("component allowlist must be enough: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadClientTLSRejectsUnappliedTargetAllowlistFromEnv(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca := newTestCA(t, dir)
|
||||
certFile, keyFile := ca.issue(t, dir, "client", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})
|
||||
|
||||
raw := viper.New()
|
||||
raw.AutomaticEnv()
|
||||
raw.SetEnvPrefix("weed")
|
||||
raw.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
|
||||
v := util.NewViperProxy(raw)
|
||||
v.Set("grpc.ca", ca.file)
|
||||
v.Set("grpc.client.cert", certFile)
|
||||
v.Set("grpc.client.key", keyFile)
|
||||
t.Setenv("WEED_GRPC_CLIENT_FILER_ALLOWED_SERVER_COMMONNAMES", "seaweed-filer")
|
||||
|
||||
if _, err := loadClientTLS(v, "grpc.client", "grpc.client"); err == nil {
|
||||
t.Fatal("expected error when a per-target allowlist comes only from the environment")
|
||||
}
|
||||
fileOnly := util.NewViperProxy(viper.New())
|
||||
fileOnly.Set("grpc.ca", ca.file)
|
||||
fileOnly.Set("grpc.client.cert", certFile)
|
||||
fileOnly.Set("grpc.client.key", keyFile)
|
||||
if _, err := loadClientTLS(fileOnly, "grpc.client", "grpc.client"); err != nil {
|
||||
t.Fatalf("a config that does not read the environment must ignore it: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadClientTLSFilerKeepsTargetAllowlists(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca := newTestCA(t, dir)
|
||||
serverCert, serverKey := ca.issue(t, dir, "seaweed-volume", []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth})
|
||||
clientCert, clientKey := ca.issue(t, dir, "client", []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth})
|
||||
|
||||
v := util.NewViperProxy(viper.New())
|
||||
v.Set("grpc.ca", ca.file)
|
||||
v.Set("grpc.volume.cert", serverCert)
|
||||
v.Set("grpc.volume.key", serverKey)
|
||||
v.Set("grpc.filer.client_cert", clientCert)
|
||||
v.Set("grpc.filer.client_key", clientKey)
|
||||
v.Set("grpc.filer.volume_allowed_server_commonNames", "seaweed-volume")
|
||||
v.Set("grpc.filer.master_allowed_server_commonNames", "seaweed-master")
|
||||
|
||||
if _, err := loadClientTLS(v, "grpc.filer", "grpc.filer"); err != nil {
|
||||
t.Fatalf("the filer applies its per-target keys through LoadClientTLSForTarget: %v", err)
|
||||
}
|
||||
addr := startTestGrpcServer(t, v, "grpc.volume")
|
||||
if err := healthCheck(t, addr, LoadClientTLSForTarget(v, "grpc.filer", "grpc.master")); err == nil {
|
||||
t.Fatal("expected the master allowlist to reject a volume certificate")
|
||||
}
|
||||
}
|
||||
@@ -683,6 +683,8 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
|
||||
aggReaderName := fmt.Sprintf("aggMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch)
|
||||
aggNotifyChan := fs.filer.MetaAggregator.MetaLogBuffer.RegisterSubscriber(aggNotifyName)
|
||||
defer fs.filer.MetaAggregator.MetaLogBuffer.UnregisterSubscriber(aggNotifyName)
|
||||
fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, subscriptionWakeFilter(req))
|
||||
defer fs.filer.MetaAggregator.MetaLogBuffer.SetSubscriberWakeFilter(aggReaderName, nil)
|
||||
|
||||
gapStall := &gapStallReporter{scope: "aggregated", clientName: clientName, pathPrefix: req.PathPrefix}
|
||||
defer gapStall.close()
|
||||
@@ -1127,6 +1129,8 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
|
||||
localReaderName := fmt.Sprintf("localMeta:%s:%d:%d", clientName, req.ClientId, req.ClientEpoch)
|
||||
localFlushChan := fs.filer.LocalMetaLogBuffer.RegisterFlushSubscriber(localNotifyName)
|
||||
defer fs.filer.LocalMetaLogBuffer.UnregisterFlushSubscriber(localNotifyName)
|
||||
fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, subscriptionWakeFilter(req))
|
||||
defer fs.filer.LocalMetaLogBuffer.SetSubscriberWakeFilter(localReaderName, nil)
|
||||
|
||||
gapStall := &gapStallReporter{scope: "local", clientName: clientName, pathPrefix: req.PathPrefix}
|
||||
defer gapStall.close()
|
||||
@@ -1305,11 +1309,32 @@ func (fs *FilerServer) subscribeLocalMetadata(req *filer_pb.SubscribeMetadataReq
|
||||
|
||||
}
|
||||
|
||||
// subscriptionWakeFilter lets a reader sleep through appends in directories
|
||||
// the subscription cannot match; it reads them on its next poll instead.
|
||||
func subscriptionWakeFilter(req *filer_pb.SubscribeMetadataRequest) func(key []byte) bool {
|
||||
if req.PathPrefix == "" {
|
||||
return nil
|
||||
}
|
||||
return func(key []byte) bool {
|
||||
return filer_pb.MetadataDirectoryMayMatchSubscription(key, req.PathPrefix, req.PathPrefixes, req.Directories)
|
||||
}
|
||||
}
|
||||
|
||||
func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest, sender metadataStreamSender, eachEventNotificationFn func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error, filtered *int64) log_buffer.EachLogEntryFuncType {
|
||||
// A shallow scan of the path fields skips unmarshaling chunk-heavy events
|
||||
// this subscriber would filter out anyway; scan surprises fall back to the
|
||||
// full decode. Only a delivery resets the shared unsynced-events counter.
|
||||
prefilter := req.PathPrefix != "" || len(req.PathPrefixes) > 0 || len(req.Directories) > 0
|
||||
mayMatch := subscriptionWakeFilter(req)
|
||||
skip := func(data []byte) bool {
|
||||
if mayMatch != nil {
|
||||
if dir, target, ok := filer_pb.ScanMetadataEventDirectories(data); ok && !mayMatch(dir) && (len(target) == 0 || !mayMatch(target)) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
skeleton, ok := filer_pb.ScanMetadataEventSkeleton(data)
|
||||
return ok && !filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories)
|
||||
}
|
||||
return func(logEntry *filer_pb.LogEntry) (bool, error) {
|
||||
// A cancelled context ends the pass here: neither loop checks it
|
||||
// per entry, and the send path alone cannot cover filtered entries.
|
||||
@@ -1317,8 +1342,7 @@ func eachLogEntryFn(ctx context.Context, req *filer_pb.SubscribeMetadataRequest,
|
||||
return true, nil
|
||||
}
|
||||
if prefilter {
|
||||
if skeleton, ok := filer_pb.ScanMetadataEventSkeleton(logEntry.Data); ok &&
|
||||
!filer_pb.MetadataEventMatchesSubscription(skeleton, req.PathPrefix, req.PathPrefixes, req.Directories) {
|
||||
if skip(logEntry.Data) {
|
||||
*filtered++
|
||||
if *filtered > MaxUnsyncedEvents {
|
||||
if err := sender.Send(&filer_pb.SubscribeMetadataResponse{
|
||||
|
||||
@@ -158,3 +158,33 @@ func TestEachLogEntryFnFilteredHeartbeatUsesStreamPosition(t *testing.T) {
|
||||
t.Fatalf("heartbeat ts = %+v, want stream position 101", sender.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEachLogEntryFnPrefilterKeepsMovesIntoPrefix(t *testing.T) {
|
||||
req := &filer_pb.SubscribeMetadataRequest{PathPrefix: "/data/pvc-7/"}
|
||||
var delivered []string
|
||||
var unsyncedEvents int64
|
||||
fn := eachLogEntryFn(context.Background(), req, &recordingSender{}, func(dirPath string, eventNotification *filer_pb.EventNotification, tsNs int64, mutationTsNs int64) error {
|
||||
delivered = append(delivered, eventNotification.NewParentPath)
|
||||
return nil
|
||||
}, &unsyncedEvents)
|
||||
|
||||
for i, target := range []string{"/data/pvc-7", "/data/pvc-1/sub"} {
|
||||
data, err := proto.Marshal(&filer_pb.SubscribeMetadataResponse{
|
||||
Directory: "/data/pvc-1",
|
||||
EventNotification: &filer_pb.EventNotification{
|
||||
OldEntry: &filer_pb.Entry{Name: "a.log"},
|
||||
NewEntry: &filer_pb.Entry{Name: "a.log"},
|
||||
NewParentPath: target,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := fn(&filer_pb.LogEntry{TsNs: int64(i + 1), Data: data}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if len(delivered) != 1 || delivered[0] != "/data/pvc-7" {
|
||||
t.Fatalf("only the move into the prefix must reach the notification fn, got %v", delivered)
|
||||
}
|
||||
}
|
||||
@@ -165,9 +165,10 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i
|
||||
buckets = append(buckets, in.Bucket)
|
||||
}
|
||||
}
|
||||
walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) error {
|
||||
persister := &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)}
|
||||
walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
return dailyrun.WalkBuckets(walkCtx, view, shards, buckets,
|
||||
dailyrun.FilerListFunc(filerClient, bucketsPath, view), walkerDispatch, nil)
|
||||
dailyrun.FilerListFunc(filerClient, bucketsPath, view), walkerDispatch, nil, persister)
|
||||
})
|
||||
return dailyrun.Run(passCtx, dailyrun.Config{
|
||||
Shards: shards,
|
||||
@@ -175,7 +176,7 @@ func (c *commandS3LifecycleRunShard) Do(args []string, env *CommandEnv, writer i
|
||||
Engine: eng,
|
||||
FilerClient: filerClient,
|
||||
Client: client,
|
||||
Persister: &dailyrun.FilerCursorPersister{Store: dispatcher.NewFilerStoreClient(filerClient)},
|
||||
Persister: persister,
|
||||
Lister: dispatcher.NewFilerSiblingLister(filerClient, bucketsPath),
|
||||
// The shell command is used for bounded one-shot sweeps in
|
||||
// integration tests and CI. Fan out across the selected shards
|
||||
|
||||
@@ -194,8 +194,11 @@ type LogBuffer struct {
|
||||
subscribers map[string]*subscription // subscriberID -> shared notification channel
|
||||
// Notified only when a flush lands, for readers that cannot act on an append
|
||||
flushSubscribers map[string]*subscription
|
||||
isStopping *atomic.Bool
|
||||
shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers
|
||||
// wakeFilters skip a subscriber's append wake-up when the entry's partition
|
||||
// key cannot interest it; it still reads the entry on its next poll.
|
||||
wakeFilters map[string]func(key []byte) bool
|
||||
isStopping *atomic.Bool
|
||||
shutdownCh chan struct{} // closed by ShutdownLogBuffer to wake blocked subscribers
|
||||
// flushDroppedBytes counts bytes the flushFn gave up on during shutdown.
|
||||
// loopFlush reads it after each window so an abandoned flush does not
|
||||
// advance the flushed watermark or wake subscribers as if it had landed.
|
||||
@@ -232,6 +235,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc
|
||||
notifyFn: notifyFn,
|
||||
subscribers: make(map[string]*subscription),
|
||||
flushSubscribers: make(map[string]*subscription),
|
||||
wakeFilters: make(map[string]func(key []byte) bool),
|
||||
flushChan: make(chan *dataToFlush, flushQueueDepth),
|
||||
isStopping: new(atomic.Bool),
|
||||
shutdownCh: make(chan struct{}),
|
||||
@@ -259,6 +263,7 @@ func NewLogBuffer(name string, flushInterval time.Duration, flushFn LogFlushFunc
|
||||
type subscription struct {
|
||||
notifyChan chan struct{}
|
||||
refCount int
|
||||
wants func(key []byte) bool
|
||||
}
|
||||
|
||||
func registerSubscription(subscriptions map[string]*subscription, subscriberID string) chan struct{} {
|
||||
@@ -292,7 +297,9 @@ func (logBuffer *LogBuffer) RegisterSubscriber(subscriberID string) chan struct{
|
||||
logBuffer.subscribersMu.Lock()
|
||||
defer logBuffer.subscribersMu.Unlock()
|
||||
|
||||
return registerSubscription(logBuffer.subscribers, subscriberID)
|
||||
notifyChan := registerSubscription(logBuffer.subscribers, subscriberID)
|
||||
logBuffer.subscribers[subscriberID].wants = logBuffer.wakeFilters[subscriberID]
|
||||
return notifyChan
|
||||
}
|
||||
|
||||
// UnregisterSubscriber removes a subscriber and closes its notification channel
|
||||
@@ -374,23 +381,42 @@ func (logBuffer *LogBuffer) IsOffsetInMemory(offset int64) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// notifySubscribers sends notifications to all registered subscribers
|
||||
// Non-blocking: uses select with default to avoid blocking on full channels
|
||||
func (logBuffer *LogBuffer) notifySubscribers() {
|
||||
// SetSubscriberWakeFilter limits append wake-ups of the subscriber registered
|
||||
// as subscriberID to entries whose partition key satisfies wants; a nil wants
|
||||
// removes the filter. Flushes and keyless appends still wake every subscriber.
|
||||
func (logBuffer *LogBuffer) SetSubscriberWakeFilter(subscriberID string, wants func(key []byte) bool) {
|
||||
logBuffer.subscribersMu.Lock()
|
||||
defer logBuffer.subscribersMu.Unlock()
|
||||
|
||||
if wants == nil {
|
||||
delete(logBuffer.wakeFilters, subscriberID)
|
||||
} else {
|
||||
logBuffer.wakeFilters[subscriberID] = wants
|
||||
}
|
||||
if sub, ok := logBuffer.subscribers[subscriberID]; ok {
|
||||
sub.wants = wants
|
||||
}
|
||||
}
|
||||
|
||||
// NotifySubscribers wakes the subscribers interested in key, for entries that
|
||||
// also concern a key other than the partition key they were appended under.
|
||||
func (logBuffer *LogBuffer) NotifySubscribers(key []byte) {
|
||||
logBuffer.notifySubscribers(key)
|
||||
}
|
||||
|
||||
// notifySubscribers wakes the subscribers interested in key, or all of them
|
||||
// for an empty key. Non-blocking: a full channel already holds a wake-up.
|
||||
func (logBuffer *LogBuffer) notifySubscribers(key []byte) {
|
||||
logBuffer.subscribersMu.RLock()
|
||||
defer logBuffer.subscribersMu.RUnlock()
|
||||
|
||||
if len(logBuffer.subscribers) == 0 {
|
||||
return // No subscribers, skip notification
|
||||
}
|
||||
|
||||
for _, sub := range logBuffer.subscribers {
|
||||
if len(key) > 0 && sub.wants != nil && !sub.wants(key) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
case sub.notifyChan <- struct{}{}:
|
||||
// Notification sent successfully
|
||||
default:
|
||||
// Channel full - subscriber hasn't consumed previous notification yet
|
||||
// This is OK because one notification is sufficient to wake the subscriber
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -468,7 +494,7 @@ func (logBuffer *LogBuffer) AddLogEntryToBuffer(logEntry *filer_pb.LogEntry) err
|
||||
logBuffer.notifyFn()
|
||||
}
|
||||
// Notify all registered subscribers instantly (<1ms latency)
|
||||
logBuffer.notifySubscribers()
|
||||
logBuffer.notifySubscribers(logEntry.Key)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -607,7 +633,7 @@ func (logBuffer *LogBuffer) AddDataToBufferAndGetTs(partitionKey, data []byte, p
|
||||
logBuffer.notifyFn()
|
||||
}
|
||||
// Notify all registered subscribers instantly (<1ms latency)
|
||||
logBuffer.notifySubscribers()
|
||||
logBuffer.notifySubscribers(partitionKey)
|
||||
}
|
||||
}()
|
||||
|
||||
@@ -823,7 +849,7 @@ func (logBuffer *LogBuffer) loopFlush() {
|
||||
if logBuffer.notifyFn != nil {
|
||||
logBuffer.notifyFn()
|
||||
}
|
||||
logBuffer.notifySubscribers()
|
||||
logBuffer.notifySubscribers(nil)
|
||||
logBuffer.notifyFlushSubscribers()
|
||||
|
||||
// Signal completion if there's a callback channel
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package log_buffer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
func woken(ch <-chan struct{}) bool {
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriberWakeFilter(t *testing.T) {
|
||||
flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}
|
||||
logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil)
|
||||
defer logBuffer.ShutdownLogBuffer()
|
||||
|
||||
wantsA := func(key []byte) bool { return bytes.Equal(key, []byte("a")) }
|
||||
// set before registration, the way a reader is set up ahead of its loop
|
||||
logBuffer.SetSubscriberWakeFilter("filtered", wantsA)
|
||||
filtered := logBuffer.RegisterSubscriber("filtered")
|
||||
defer logBuffer.UnregisterSubscriber("filtered")
|
||||
all := logBuffer.RegisterSubscriber("all")
|
||||
defer logBuffer.UnregisterSubscriber("all")
|
||||
late := logBuffer.RegisterSubscriber("late")
|
||||
defer logBuffer.UnregisterSubscriber("late")
|
||||
logBuffer.SetSubscriberWakeFilter("late", wantsA)
|
||||
|
||||
if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if woken(filtered) || woken(late) {
|
||||
t.Fatal("an append under another key must not wake filtered subscribers")
|
||||
}
|
||||
if !woken(all) {
|
||||
t.Fatal("an unfiltered subscriber must wake on every append")
|
||||
}
|
||||
|
||||
if err := logBuffer.AddDataToBuffer([]byte("a"), []byte("v"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !woken(filtered) || !woken(late) || !woken(all) {
|
||||
t.Fatal("an append under a wanted key must wake every subscriber")
|
||||
}
|
||||
|
||||
logBuffer.NotifySubscribers([]byte("a"))
|
||||
if !woken(filtered) {
|
||||
t.Fatal("NotifySubscribers must wake subscribers that want the key")
|
||||
}
|
||||
|
||||
logBuffer.ForceFlush()
|
||||
if !woken(filtered) || !woken(late) {
|
||||
t.Fatal("a flush must wake filtered subscribers")
|
||||
}
|
||||
|
||||
logBuffer.SetSubscriberWakeFilter("late", nil)
|
||||
if err := logBuffer.AddDataToBuffer([]byte("b"), []byte("v"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !woken(late) {
|
||||
t.Fatal("clearing the filter must restore wake-ups")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSubscriberWakeFilterStillDelivers(t *testing.T) {
|
||||
flushFn := func(*LogBuffer, time.Time, time.Time, []byte, int64, int64) {}
|
||||
logBuffer := NewLogBuffer("test", time.Minute, flushFn, nil, nil)
|
||||
defer logBuffer.ShutdownLogBuffer()
|
||||
|
||||
logBuffer.SetSubscriberWakeFilter("reader", func([]byte) bool { return false })
|
||||
start := NewMessagePosition(time.Now().UnixNano(), -2)
|
||||
got := make(chan []byte, 1)
|
||||
go logBuffer.LoopProcessLogData("reader", start, 0, func() bool { return true }, func(logEntry *filer_pb.LogEntry) (bool, error) {
|
||||
got <- bytes.Clone(logEntry.Data)
|
||||
return true, nil
|
||||
})
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
if err := logBuffer.AddDataToBuffer([]byte("unwanted"), []byte("v"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
select {
|
||||
case data := <-got:
|
||||
if string(data) != "v" {
|
||||
t.Fatalf("got %q", data)
|
||||
}
|
||||
case <-time.After(2 * notificationHealthCheckInterval):
|
||||
t.Fatal("a reader that sleeps through an append must still read it on its next poll")
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
var (
|
||||
MAJOR_VERSION = int32(4)
|
||||
MINOR_VERSION = int32(53)
|
||||
MINOR_VERSION = int32(54)
|
||||
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
|
||||
VERSION = util.SizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
|
||||
@@ -42,6 +42,9 @@ const (
|
||||
// the per-shard walk budget — typically 60 (1h) for small clusters,
|
||||
// 360+ (6h+) for large ones.
|
||||
WalkerIntervalMinutesAdminKey = "walker_interval_minutes"
|
||||
// DeleteTimeoutSecondsAdminKey bounds each LifecycleDelete attempt.
|
||||
// 0 = dailyrun.DefaultDeleteTimeout.
|
||||
DeleteTimeoutSecondsAdminKey = "delete_timeout_seconds"
|
||||
|
||||
// MetadataKeyDeletesPerSecond is the per-worker share value the
|
||||
// admin writes into ClusterContext.Metadata at ExecuteJob time.
|
||||
|
||||
@@ -21,6 +21,9 @@ type Config struct {
|
||||
// behavior; positive values gate the walker via Cursor.LastWalkedNs
|
||||
// inside dailyrun.runShard.
|
||||
WalkerInterval time.Duration
|
||||
// DeleteTimeout bounds each LifecycleDelete attempt; 0 uses
|
||||
// dailyrun.DefaultDeleteTimeout.
|
||||
DeleteTimeout time.Duration
|
||||
}
|
||||
|
||||
func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map[string]*plugin_pb.ConfigValue) Config {
|
||||
@@ -42,6 +45,9 @@ func ParseConfig(adminValues map[string]*plugin_pb.ConfigValue, workerValues map
|
||||
if mins := readInt64(adminValues, WalkerIntervalMinutesAdminKey, 0); mins > 0 {
|
||||
cfg.WalkerInterval = time.Duration(mins) * time.Minute
|
||||
}
|
||||
if secs := readInt64(adminValues, DeleteTimeoutSecondsAdminKey, 0); secs > 0 {
|
||||
cfg.DeleteTimeout = time.Duration(secs) * time.Second
|
||||
}
|
||||
return cfg
|
||||
}
|
||||
|
||||
|
||||
@@ -93,3 +93,21 @@ func TestParseConfigWalkerIntervalNegativeStaysZero(t *testing.T) {
|
||||
t.Errorf("negative WalkerInterval should stay 0, got %v", cfg.WalkerInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseConfig_DeleteTimeout(t *testing.T) {
|
||||
if cfg := ParseConfig(nil, nil); cfg.DeleteTimeout != 0 {
|
||||
t.Errorf("DeleteTimeout default=%v, want 0 (dailyrun default)", cfg.DeleteTimeout)
|
||||
}
|
||||
cfg := ParseConfig(map[string]*plugin_pb.ConfigValue{
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 45}},
|
||||
}, nil)
|
||||
if want := 45 * time.Second; cfg.DeleteTimeout != want {
|
||||
t.Errorf("DeleteTimeout=%v, want %v", cfg.DeleteTimeout, want)
|
||||
}
|
||||
cfg = ParseConfig(map[string]*plugin_pb.ConfigValue{
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: -5}},
|
||||
}, nil)
|
||||
if cfg.DeleteTimeout != 0 {
|
||||
t.Errorf("negative DeleteTimeout should stay 0, got %v", cfg.DeleteTimeout)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package s3_lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// failoverClient sends LifecycleDelete to one S3 front and, when that front
|
||||
// becomes unreachable, moves the pass to the next front that passes
|
||||
// checkLifecycleReady. A front that failed, including one selectS3Front
|
||||
// skipped before addr, is not selected again in the pass.
|
||||
type failoverClient struct {
|
||||
ctx context.Context
|
||||
h *Handler
|
||||
endpoints []string
|
||||
report func(string)
|
||||
|
||||
mu sync.Mutex
|
||||
conn *grpc.ClientConn
|
||||
addr string
|
||||
client lifecycleRPCAdapter
|
||||
failed map[string]bool
|
||||
exhausted error
|
||||
}
|
||||
|
||||
func (h *Handler) newFailoverClient(ctx context.Context, conn *grpc.ClientConn, addr string, endpoints []string, report func(string)) *failoverClient {
|
||||
f := &failoverClient{ctx: ctx, h: h, endpoints: endpoints, report: report, failed: map[string]bool{}}
|
||||
for _, e := range endpoints {
|
||||
if e == addr {
|
||||
break
|
||||
}
|
||||
f.failed[e] = true
|
||||
}
|
||||
f.use(conn, addr)
|
||||
return f
|
||||
}
|
||||
|
||||
func (f *failoverClient) use(conn *grpc.ClientConn, addr string) {
|
||||
f.conn, f.addr = conn, addr
|
||||
f.client = lifecycleRPCAdapter{c: s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(conn)}
|
||||
}
|
||||
|
||||
// LifecycleDelete resends the same request after each switch: the server's
|
||||
// identity check resolves a delete that landed before the reply was lost
|
||||
// instead of applying it twice. Every switch retires a front, which bounds
|
||||
// the loop.
|
||||
func (f *failoverClient) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
for {
|
||||
f.mu.Lock()
|
||||
conn, client, exhausted := f.conn, f.client, f.exhausted
|
||||
f.mu.Unlock()
|
||||
if exhausted != nil {
|
||||
return nil, exhausted
|
||||
}
|
||||
resp, err := client.LifecycleDelete(ctx, req)
|
||||
if code := status.Code(err); err == nil || ctx.Err() != nil || (code != codes.Unavailable && code != codes.Canceled) {
|
||||
return resp, err
|
||||
}
|
||||
if err := f.failover(conn, err); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// failover replaces the front behind failing when it is unavailable; a call
|
||||
// cut off by an earlier switch closing its connection just moves on.
|
||||
func (f *failoverClient) failover(failing *grpc.ClientConn, cause error) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.exhausted != nil {
|
||||
return f.exhausted
|
||||
}
|
||||
if f.conn != failing {
|
||||
return nil
|
||||
}
|
||||
code := status.Code(cause)
|
||||
if code != codes.Unavailable {
|
||||
return cause
|
||||
}
|
||||
from := f.addr
|
||||
f.failed[from] = true
|
||||
f.conn.Close()
|
||||
f.conn = nil
|
||||
|
||||
var remaining []string
|
||||
for _, addr := range f.endpoints {
|
||||
if !f.failed[addr] {
|
||||
remaining = append(remaining, addr)
|
||||
}
|
||||
}
|
||||
conn, addr, skipped := f.h.selectS3Front(f.ctx, remaining)
|
||||
if f.ctx.Err() == nil {
|
||||
for _, s := range remaining {
|
||||
if s == addr {
|
||||
break
|
||||
}
|
||||
f.failed[s] = true
|
||||
}
|
||||
}
|
||||
lost := fmt.Errorf("s3 %s: %s", from, code)
|
||||
if conn == nil {
|
||||
f.exhausted = noReadyS3Error(append([]error{lost}, skipped...))
|
||||
glog.Warningf("s3 lifecycle: %v", f.exhausted)
|
||||
f.report(f.exhausted.Error())
|
||||
return f.exhausted
|
||||
}
|
||||
f.use(conn, addr)
|
||||
msg := fmt.Sprintf("switched to s3 %s, %v", addr, lost)
|
||||
if len(skipped) > 0 {
|
||||
msg += fmt.Sprintf(", skipped: %v", errors.Join(skipped...))
|
||||
}
|
||||
glog.Warningf("s3 lifecycle: %s", msg)
|
||||
f.report(msg)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *failoverClient) Close() {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.conn != nil {
|
||||
f.conn.Close()
|
||||
f.conn = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,312 @@
|
||||
package s3_lifecycle
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_lifecycle_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
type failoverFront struct {
|
||||
testFront
|
||||
dieOn int
|
||||
hold bool
|
||||
stop func()
|
||||
|
||||
mu sync.Mutex
|
||||
reqs []*s3_lifecycle_pb.LifecycleDeleteRequest
|
||||
}
|
||||
|
||||
// LifecycleDelete applies the request; on the dieOn-th call the server then
|
||||
// stops before replying, like a front lost after the delete landed.
|
||||
func (f *failoverFront) LifecycleDelete(ctx context.Context, req *s3_lifecycle_pb.LifecycleDeleteRequest) (*s3_lifecycle_pb.LifecycleDeleteResponse, error) {
|
||||
if f.hold {
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
resp, err := f.testFront.LifecycleDelete(ctx, req)
|
||||
f.mu.Lock()
|
||||
f.reqs = append(f.reqs, proto.Clone(req).(*s3_lifecycle_pb.LifecycleDeleteRequest))
|
||||
n := len(f.reqs)
|
||||
f.mu.Unlock()
|
||||
if n == f.dieOn {
|
||||
go f.stop()
|
||||
<-ctx.Done()
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (f *failoverFront) received() []*s3_lifecycle_pb.LifecycleDeleteRequest {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]*s3_lifecycle_pb.LifecycleDeleteRequest(nil), f.reqs...)
|
||||
}
|
||||
|
||||
func startFailoverFront(t *testing.T, f *failoverFront) string {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := grpc.NewServer()
|
||||
s3_lifecycle_pb.RegisterSeaweedS3LifecycleInternalServer(s, f)
|
||||
f.stop = s.Stop
|
||||
go s.Serve(lis)
|
||||
t.Cleanup(s.Stop)
|
||||
return lis.Addr().String()
|
||||
}
|
||||
|
||||
func downAddr(t *testing.T) string {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addr := lis.Addr().String()
|
||||
lis.Close()
|
||||
return addr
|
||||
}
|
||||
|
||||
type reports struct {
|
||||
mu sync.Mutex
|
||||
msgs []string
|
||||
}
|
||||
|
||||
func (r *reports) add(msg string) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
r.msgs = append(r.msgs, msg)
|
||||
}
|
||||
|
||||
func (r *reports) all() []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
return append([]string(nil), r.msgs...)
|
||||
}
|
||||
|
||||
func newTestFailoverClient(t *testing.T, ctx context.Context, endpoints []string, r *reports) *failoverClient {
|
||||
t.Helper()
|
||||
h := newTestHandler()
|
||||
conn, addr, skipped := h.selectS3Front(ctx, endpoints)
|
||||
if conn == nil {
|
||||
t.Fatalf("no ready front: %v", skipped)
|
||||
}
|
||||
f := h.newFailoverClient(ctx, conn, addr, endpoints, r.add)
|
||||
t.Cleanup(f.Close)
|
||||
return f
|
||||
}
|
||||
|
||||
func deleteReq(i int) *s3_lifecycle_pb.LifecycleDeleteRequest {
|
||||
return &s3_lifecycle_pb.LifecycleDeleteRequest{
|
||||
Bucket: "bk",
|
||||
ObjectPath: fmt.Sprintf("secret-key-%d", i),
|
||||
VersionId: fmt.Sprintf("v%d", i),
|
||||
RuleHash: []byte{1, 2, 3},
|
||||
ActionKind: s3_lifecycle_pb.ActionKind_NONCURRENT_DAYS,
|
||||
ExpectedIdentity: &s3_lifecycle_pb.EntryIdentity{MtimeNs: int64(i), Size: 7, HeadFid: "3,01"},
|
||||
}
|
||||
}
|
||||
|
||||
func TestSelectS3Front_FirstFrontDownUsesNext(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
down := downAddr(t)
|
||||
goodAddr := startFront(t, &testFront{key: frontKey})
|
||||
|
||||
conn, got, skipped := newTestHandler().selectS3Front(context.Background(), []string{down, goodAddr})
|
||||
if conn == nil {
|
||||
t.Fatalf("expected the second front, skipped: %v", skipped)
|
||||
}
|
||||
conn.Close()
|
||||
if got != goodAddr || len(skipped) != 1 || !strings.Contains(skipped[0].Error(), down) {
|
||||
t.Fatalf("got %q skipped %v, want %q after skipping %s", got, skipped, goodAddr, down)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_FrontLostMidPassResendsSameRequest(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 3}
|
||||
b := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
aAddr, bAddr := startFailoverFront(t, a), startFailoverFront(t, b)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr}, r)
|
||||
|
||||
for i := 1; i <= 5; i++ {
|
||||
resp, err := f.LifecycleDelete(context.Background(), deleteReq(i))
|
||||
if err != nil || resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_DONE {
|
||||
t.Fatalf("delete %d: resp %v err %v", i, resp, err)
|
||||
}
|
||||
}
|
||||
gotA, gotB := a.received(), b.received()
|
||||
if len(gotA) != 3 || len(gotB) != 3 {
|
||||
t.Fatalf("front a got %d, front b got %d deletes; want 3 and 3", len(gotA), len(gotB))
|
||||
}
|
||||
if !proto.Equal(gotA[2], gotB[0]) || !proto.Equal(gotB[0], deleteReq(3)) {
|
||||
t.Fatalf("resent request differs: a=%v b=%v", gotA[2], gotB[0])
|
||||
}
|
||||
msgs := r.all()
|
||||
if len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+bAddr) || !strings.Contains(msgs[0], aAddr) || !strings.Contains(msgs[0], "Unavailable") {
|
||||
t.Fatalf("want one switch report naming both fronts, got %q", msgs)
|
||||
}
|
||||
for _, m := range msgs {
|
||||
assertNoSecrets(t, m)
|
||||
if strings.Contains(m, "secret-key") {
|
||||
t.Fatalf("report exposes an object key: %s", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_ConcurrentCallersSwitchOnce(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 4}
|
||||
b := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
c := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
aAddr, bAddr, cAddr := startFailoverFront(t, a), startFailoverFront(t, b), startFailoverFront(t, c)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr, cAddr}, r)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, 40)
|
||||
for i := 0; i < 40; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
if _, err := f.LifecycleDelete(context.Background(), deleteReq(i)); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
t.Errorf("delete failed: %v", err)
|
||||
}
|
||||
if msgs := r.all(); len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+bAddr) {
|
||||
t.Fatalf("want one switch to %s, got %q", bAddr, msgs)
|
||||
}
|
||||
if n := len(c.received()); n != 0 {
|
||||
t.Fatalf("front c got %d deletes, want none", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_NoFrontLeftFailsFast(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
|
||||
aAddr := startFailoverFront(t, a)
|
||||
down := downAddr(t)
|
||||
rejecting := &testFront{key: otherKey}
|
||||
rejectAddr := startFront(t, rejecting)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{aAddr, down, rejectAddr}, r)
|
||||
|
||||
start := time.Now()
|
||||
_, err := f.LifecycleDelete(context.Background(), deleteReq(1))
|
||||
if err == nil {
|
||||
t.Fatal("want an error with no front left")
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, want := range []string{"no s3 server is ready for lifecycle deletes", aAddr, down, rejectAddr, "rejected lifecycle credentials", "jwt.filer_signing.key"} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Fatalf("error %q does not mention %q", msg, want)
|
||||
}
|
||||
}
|
||||
assertNoSecrets(t, msg)
|
||||
if strings.Contains(msg, "secret-key") {
|
||||
t.Fatalf("error exposes an object key: %s", msg)
|
||||
}
|
||||
|
||||
if _, err2 := f.LifecycleDelete(context.Background(), deleteReq(2)); err2 == nil || err2.Error() != msg {
|
||||
t.Fatalf("second call: want the same error, got %v", err2)
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 10*time.Second {
|
||||
t.Fatalf("giving up took %v", elapsed)
|
||||
}
|
||||
if n := rejecting.readies.Load(); n != 1 {
|
||||
t.Fatalf("rejecting front checked %d times, want once", n)
|
||||
}
|
||||
if msgs := r.all(); len(msgs) != 1 || msgs[0] != msg {
|
||||
t.Fatalf("want the error reported once, got %q", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_ParentCancelDoesNotSwitch(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
a := &failoverFront{testFront: testFront{key: frontKey}, hold: true}
|
||||
b := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
aAddr, bAddr := startFailoverFront(t, a), startFailoverFront(t, b)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr}, r)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
time.AfterFunc(100*time.Millisecond, cancel)
|
||||
start := time.Now()
|
||||
if _, err := f.LifecycleDelete(ctx, deleteReq(1)); err == nil {
|
||||
t.Fatal("want an error after cancellation")
|
||||
}
|
||||
if elapsed := time.Since(start); elapsed > 5*time.Second {
|
||||
t.Fatalf("cancelled delete took %v", elapsed)
|
||||
}
|
||||
if b.readies.Load() != 0 || len(r.all()) != 0 {
|
||||
t.Fatalf("switched fronts after cancellation: %d readiness checks, reports %q", b.readies.Load(), r.all())
|
||||
}
|
||||
f.mu.Lock()
|
||||
current := f.addr
|
||||
f.mu.Unlock()
|
||||
if current != aAddr {
|
||||
t.Fatalf("current front %q, want %q", current, aAddr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_SecondFrontLostMovesToThird(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
a := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
|
||||
b := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
|
||||
c := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
aAddr, bAddr, cAddr := startFailoverFront(t, a), startFailoverFront(t, b), startFailoverFront(t, c)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{aAddr, bAddr, cAddr}, r)
|
||||
|
||||
resp, err := f.LifecycleDelete(context.Background(), deleteReq(1))
|
||||
if err != nil || resp.Outcome != s3_lifecycle_pb.LifecycleDeleteOutcome_DONE {
|
||||
t.Fatalf("resp %v err %v", resp, err)
|
||||
}
|
||||
for name, front := range map[string]*failoverFront{"a": a, "b": b, "c": c} {
|
||||
got := front.received()
|
||||
if len(got) != 1 || !proto.Equal(got[0], deleteReq(1)) {
|
||||
t.Fatalf("front %s got %v, want the one request", name, got)
|
||||
}
|
||||
}
|
||||
if msgs := r.all(); len(msgs) != 2 || !strings.Contains(msgs[1], "switched to s3 "+cAddr) {
|
||||
t.Fatalf("want switches to b then c, got %q", msgs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverClient_SkipsFrontSkippedAtStartup(t *testing.T) {
|
||||
setWorkerKey(t, frontKey)
|
||||
prev := lifecycleReadyTimeout
|
||||
lifecycleReadyTimeout = 200 * time.Millisecond
|
||||
t.Cleanup(func() { lifecycleReadyTimeout = prev })
|
||||
stuck := &failoverFront{testFront: testFront{key: frontKey, block: true}}
|
||||
b := &failoverFront{testFront: testFront{key: frontKey}, dieOn: 1}
|
||||
c := &failoverFront{testFront: testFront{key: frontKey}}
|
||||
stuckAddr, bAddr, cAddr := startFailoverFront(t, stuck), startFailoverFront(t, b), startFailoverFront(t, c)
|
||||
r := &reports{}
|
||||
f := newTestFailoverClient(t, context.Background(), []string{stuckAddr, bAddr, cAddr}, r)
|
||||
|
||||
if _, err := f.LifecycleDelete(context.Background(), deleteReq(1)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if n := stuck.readies.Load(); n != 1 {
|
||||
t.Fatalf("front skipped at startup checked %d times, want once", n)
|
||||
}
|
||||
if msgs := r.all(); len(msgs) != 1 || !strings.Contains(msgs[0], "switched to s3 "+cAddr) || strings.Contains(msgs[0], "skipped") {
|
||||
t.Fatalf("want a direct switch to %s, got %q", cAddr, msgs)
|
||||
}
|
||||
}
|
||||
@@ -111,6 +111,14 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
{
|
||||
Name: DeleteTimeoutSecondsAdminKey,
|
||||
Label: "Delete RPC Timeout (seconds)",
|
||||
Description: "Deadline for each lifecycle delete RPC to an S3 server, counted after the rate-limit wait. Replay and walker deletes get up to 3 attempts. After that a replay shard stops at that event and resumes there on the next run, and a walk fails and runs again from the start on the next run. 0 = 60 seconds.",
|
||||
FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64,
|
||||
Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER,
|
||||
MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -119,6 +127,7 @@ func (h *Handler) Descriptor() *plugin_pb.JobTypeDescriptor {
|
||||
ClusterDeletesBurstAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
MetaLogRetentionDaysAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
WalkerIntervalMinutesAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
DeleteTimeoutSecondsAdminKey: {Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}},
|
||||
},
|
||||
},
|
||||
// WorkerConfigForm intentionally absent — the prior
|
||||
@@ -254,7 +263,14 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
|
||||
if s3Conn == nil {
|
||||
return noReadyS3Error(skipped)
|
||||
}
|
||||
defer s3Conn.Close()
|
||||
client := h.newFailoverClient(runCtx, s3Conn, s3Endpoint, s3Endpoints, func(msg string) {
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
JobId: request.Job.JobId, JobType: jobType,
|
||||
State: plugin_pb.JobState_JOB_STATE_RUNNING, Stage: "running",
|
||||
Message: msg,
|
||||
})
|
||||
})
|
||||
defer client.Close()
|
||||
if len(skipped) > 0 {
|
||||
skippedErr := errors.Join(skipped...)
|
||||
glog.Warningf("s3 lifecycle: using s3 %s, skipped: %v", s3Endpoint, skippedErr)
|
||||
@@ -264,9 +280,8 @@ func (h *Handler) Execute(ctx context.Context, request *plugin_pb.ExecuteJobRequ
|
||||
Message: fmt.Sprintf("using s3 %s, skipped: %v", s3Endpoint, skippedErr),
|
||||
})
|
||||
}
|
||||
rpc := s3_lifecycle_pb.NewSeaweedS3LifecycleInternalClient(s3Conn)
|
||||
|
||||
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, rpc, cfg, sender)
|
||||
report, err := h.executeDailyReplay(runCtx, request, bucketsPath, filerClient, client, cfg, sender)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -296,7 +311,7 @@ func sendCompletion(request *plugin_pb.ExecuteJobRequest, sender pluginworker.Ex
|
||||
// and against the steady-state walk view; all rule kinds are serviced.
|
||||
// The returned error covers setup only; the pass's own error is in the
|
||||
// report.
|
||||
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, rpc s3_lifecycle_pb.SeaweedS3LifecycleInternalClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
|
||||
func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.ExecuteJobRequest, bucketsPath string, filerClient filer_pb.SeaweedFilerClient, client dailyrun.LifecycleClient, cfg Config, sender pluginworker.ExecutionSender) (*runReport, error) {
|
||||
startedAt := time.Now()
|
||||
eng := engine.New()
|
||||
inputs, parseErrors, err := scheduler.LoadCompileInputs(ctx, filerClient, bucketsPath)
|
||||
@@ -315,10 +330,6 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
|
||||
limiter, limiterDesc := buildLimiterFromClusterContext(request.GetClusterContext())
|
||||
|
||||
// Reuse one LifecycleClient across the replay drain and the
|
||||
// walker's per-entry dispatch.
|
||||
client := lifecycleRPCAdapter{c: rpc}
|
||||
|
||||
// Bucket list for the walker — derived from inputs so it matches
|
||||
// the snapshot the engine compiled.
|
||||
buckets := make([]string, 0, len(inputs))
|
||||
@@ -334,10 +345,10 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
runStats := &dailyrun.RunStats{}
|
||||
// Share the limiter with processMatches so walker + replay can't
|
||||
// combine to burst past the cluster cap.
|
||||
walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, Stats: runStats}
|
||||
walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) error {
|
||||
walkerDispatch := &dailyrun.WalkerDispatcher{Client: client, Limiter: limiter, DeleteTimeout: cfg.DeleteTimeout, Stats: runStats}
|
||||
walker := dailyrun.WalkerFunc(func(walkCtx context.Context, view *engine.Snapshot, shards []int) (time.Time, error) {
|
||||
return dailyrun.WalkBuckets(walkCtx, view, shards, buckets,
|
||||
dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats)
|
||||
dailyrun.FilerListFunc(runStats.CountListed(filerClient), bucketsPath, view), walkerDispatch, runStats, persister)
|
||||
})
|
||||
|
||||
_ = sender.SendProgress(&plugin_pb.JobProgressUpdate{
|
||||
@@ -358,6 +369,7 @@ func (h *Handler) executeDailyReplay(ctx context.Context, request *plugin_pb.Exe
|
||||
Lister: dispatcher.NewFilerSiblingLister(filerClient, bucketsPath),
|
||||
Workers: cfg.Workers,
|
||||
Limiter: limiter,
|
||||
DeleteTimeout: cfg.DeleteTimeout,
|
||||
RetentionWindow: cfg.MetaLogRetention,
|
||||
Walker: walker,
|
||||
WalkerInterval: cfg.WalkerInterval,
|
||||
@@ -463,9 +475,9 @@ func parsePositiveInt(s string) (int, bool) {
|
||||
}
|
||||
|
||||
// clusterS3Endpoints returns the master-discovered S3 gRPC addresses for the
|
||||
// cluster. The handler uses the first one that passes checkLifecycleReady;
|
||||
// the master refreshes the list on KeepConnected so a stale entry self-heals
|
||||
// on the next run.
|
||||
// cluster. The handler uses the first one that passes checkLifecycleReady and
|
||||
// moves to the next ready one if it becomes unavailable during the pass; the
|
||||
// master refreshes the list on KeepConnected.
|
||||
func clusterS3Endpoints(cc *plugin_pb.ClusterContext) []string {
|
||||
if cc == nil {
|
||||
return nil
|
||||
|
||||
Reference in new issue
Block a user