From 27fa285d48482d8930ed753d6de1302aacee27ad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 02:04:27 -0700 Subject: [PATCH 01/70] filer: do not stall startup waiting out the leader-settle window (#11972) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * filer: do not stall startup waiting out the leader-settle window ListExistingPeerUpdates re-polled the leader for up to 10s when it reported zero filers. A fresh single-filer cluster can never gain peers, so every filer start paid the full settle timeout before listening — past the 6s health window in weed mini and short probe budgets. Take the leader's first answer instead; the fallback fullest-answer scan below still covers a leader that is mid-convergence. * filer: bound leader-settle re-poll below the startup health window * filer: update ListExistingPeerUpdates doc for empty-leader fallback * filer: re-check empty membership at the announce gate before registering A fresh filer that scans empty membership during a master restart or re-election skipped bootstrap and announced immediately, serving missing files until a peer happened to reconnect — and a late peer only got an incremental subscription, so pre-existing metadata was never backfilled. The announce gate now re-polls ListExistingPeerUpdates while the scan can still be disproved (10s), backfills from any late peers before their streams start, and then waits for their catch-up like startup peers. A genuinely empty cluster pays a delayed master listing rather than a stalled startup, since this runs after the ports already serve. * filer: overlap the late-peer copy cutoff and stream start A peer event stamped just behind this filer's clock could land after the traversal read but carry a timestamp below the stream start — the startup scan margins both cutoffs by LogFlushInterval for exactly this skew; mirror it for peers discovered at the announce gate. * filer: bound the late-peer bootstrap copy by the shared announce deadline MaybeBootstrapFromOnePeer now takes the caller's context so a stalled peer metadata copy cannot out-wait the announce cap, and the bootstrap re-check shares one announceCatchUpCap deadline with the catch-up wait instead of each spending a full minute. * filer: merge bootstrap entries by commit time instead of blind insert The late-peer copy runs while gRPC already serves, so a blind InsertEntry could replace a write the local filer just accepted. Routing each entry through applyTraversedEntry orders it against the local commit time under the replay locks — a fresher local write survives the backfill. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/filer/filer.go | 27 +++++---- weed/pb/filer_pb/filer_client_bfs.go | 6 +- weed/server/filer_server.go | 83 ++++++++++++++++++++++++++-- 3 files changed, 97 insertions(+), 19 deletions(-) diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 522df042a9..5a0090e61b 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -190,7 +190,7 @@ func (f *Filer) buildRemoteStorageClient(ctx context.Context, remoteConf *remote return remote_storage.GetRemoteStorage(remoteConf) } -func (f *Filer) MaybeBootstrapFromOnePeer(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { +func (f *Filer) MaybeBootstrapFromOnePeer(ctx context.Context, self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, snapshotTime time.Time) (err error) { if len(existingNodes) == 0 { return } @@ -205,8 +205,13 @@ func (f *Filer) MaybeBootstrapFromOnePeer(self pb.ServerAddress, existingNodes [ glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId) return pb.WithFilerClient(false, f.UniqueFilerId, pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error { - return filer_pb.StreamBfs(client, "/", snapshotTime.UnixNano(), func(parentPath util.FullPath, entry *filer_pb.Entry) error { - return f.Store.InsertEntry(context.Background(), FromPbEntry(string(parentPath), entry)) + return filer_pb.StreamBfs(ctx, client, "/", snapshotTime.UnixNano(), func(parentPath util.FullPath, pbEntry *filer_pb.Entry, entryTsNs int64) error { + entry := FromPbEntry(string(parentPath), pbEntry) + peerTsNs := entryTsNs + if peerTsNs == 0 { + peerTsNs = entry.Attr.Mtime.UnixNano() + } + return f.applyTraversedEntry(ctx, entry, peerTsNs) }) }) @@ -271,19 +276,19 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste const peerLookupTimeout = 10 * time.Second // leaderSettleTimeout bounds how long an empty peer list from the leader is -// re-polled: a freshly elected leader has not yet received the filers' -// reconnects, so an immediate empty answer can just mean membership has not -// migrated yet. A persistent empty answer is then checked against the other -// masters below before it is accepted as an empty cluster. -const leaderSettleTimeout = 10 * time.Second +// re-polled: a freshly elected leader may not yet have the filers' reconnects, +// so an immediate empty answer can just mean membership has not migrated yet. +// It must stay below the health window a fresh filer gets to start serving +// (weed mini allows 6s) — a filerless cluster pays the whole window per start. +const leaderSettleTimeout = 3 * time.Second const leaderSettleInterval = time.Second // ListExistingPeerUpdates returns the peer list to bootstrap from. The // leader's list is authoritative: after a leader change an old master keeps // its registrations until its streams end, so the longest answer is not -// necessarily the current one. The leader's answer wins whenever it can be -// found and queried; only when the leader is unreachable does the scan fall -// back to the fullest answer from the other masters. It returns an error when +// necessarily the current one. A non-empty answer from the leader wins; a +// leader that is unreachable or reports an empty list falls back to the +// fullest answer from the other masters. It returns an error when // no master answers, so a failed lookup is not mistaken for an empty cluster. func (f *Filer) ListExistingPeerUpdates(ctx context.Context) (existingNodes []*master_pb.ClusterNodeUpdate, err error) { masters := f.MasterClient.ListMasters() diff --git a/weed/pb/filer_pb/filer_client_bfs.go b/weed/pb/filer_pb/filer_client_bfs.go index e4ec78d675..05c2a4387b 100644 --- a/weed/pb/filer_pb/filer_client_bfs.go +++ b/weed/pb/filer_pb/filer_client_bfs.go @@ -115,9 +115,9 @@ func processOneDirectory(ctx context.Context, filerClient FilerClient, parentPat } -func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry) error) (err error) { +func StreamBfs(ctx context.Context, client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry, entryTsNs int64) error) (err error) { glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs)) - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithCancel(ctx) defer cancel() stream, err := client.TraverseBfsMetadata(ctx, &TraverseBfsMetadataRequest{ Directory: string(dir), @@ -133,7 +133,7 @@ func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64 } return fmt.Errorf("traverse bfs metadata: %w", err) } - if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil { + if err := fn(util.FullPath(resp.Directory), resp.Entry, resp.EntryTsNs); err != nil { return err } } diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 41c7256172..8e09191723 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -176,6 +176,12 @@ type FilerServer struct { // replayed backlogs before the filer joins the master's list. startupPeers []pb.ServerAddress + // peerBootstrapPending marks a fresh store whose startup peer scan came + // back empty: an empty answer during a master restart can just mean the + // existing filers have not reconnected yet, so AnnounceOnceCaughtUp + // re-checks membership before registering. + peerBootstrapPending bool + // entryLockTable serializes mutations to the same entry path on this filer. // CreateEntry takes it today; UpdateEntry and DeleteEntry are intended to take // it too as their callers route a key's writes to this node, making it the @@ -367,9 +373,12 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) startFromTime := time.Now().Add(-filer.LogFlushInterval) if isFresh { glog.V(0).Infof("%s bootstrap from peers %+v", option.Host, existingNodes) - if err := fs.filer.MaybeBootstrapFromOnePeer(option.Host, existingNodes, startFromTime); err != nil { + if err := fs.filer.MaybeBootstrapFromOnePeer(context.Background(), option.Host, existingNodes, startFromTime); err != nil { glog.Fatalf("%s bootstrap from %+v: %v", option.Host, existingNodes, err) } + // An empty scan can be a settling master rather than an empty cluster; + // the announce gate then re-checks membership before registering. + fs.peerBootstrapPending = len(existingNodes) == 0 } v.SetDefault("filer.options.s3.empty_folder_cleanup_delay", "2m") if d, err := time.ParseDuration(v.GetString("filer.options.s3.empty_folder_cleanup_delay")); err == nil { @@ -494,16 +503,26 @@ const announceCatchUpCap = time.Minute // routed traffic is answerable the moment the filer is listed. func (fs *FilerServer) AnnounceOnceCaughtUp(announce chan struct{}, waitStart time.Time) { defer close(announce) - if len(fs.startupPeers) == 0 || fs.filer.MetaAggregator == nil { + ma := fs.filer.MetaAggregator + if ma == nil { return } + // The bootstrap re-check and the catch-up wait share one deadline so a + // late peer cannot stretch the registration hold past the cap. deadline := time.Now().Add(announceCatchUpCap) + peers := fs.startupPeers + if fs.peerBootstrapPending { + peers = append(peers, fs.awaitPeerBootstrap(deadline)...) + } + if len(peers) == 0 { + return + } for { // Take the channel before reading the watermarks so a rise in // between still wakes this wait (see DeliveryWatermarkAdvancedChan). - advanced := fs.filer.MetaAggregator.DeliveryWatermarkAdvancedChan() - if fs.filer.MetaAggregator.PeersCaughtUpTo(fs.startupPeers, waitStart.UnixNano()) { - glog.V(0).Infof("%s caught up with %d startup peers, announcing to masters", fs.option.Host, len(fs.startupPeers)) + advanced := ma.DeliveryWatermarkAdvancedChan() + if ma.PeersCaughtUpTo(peers, waitStart.UnixNano()) { + glog.V(0).Infof("%s caught up with %d startup peers, announcing to masters", fs.option.Host, len(peers)) return } remaining := time.Until(deadline) @@ -518,6 +537,60 @@ func (fs *FilerServer) AnnounceOnceCaughtUp(announce chan struct{}, waitStart ti } } +// emptyMembershipConfirmTimeout bounds the announce hold for a peer to +// disprove an empty startup scan on a fresh store. This runs after the ports +// already serve, so a genuinely empty cluster pays a delayed master listing, +// not a stalled startup. +const emptyMembershipConfirmTimeout = 10 * time.Second + +// awaitPeerBootstrap re-polls membership while a fresh store's empty startup +// scan can still be disproved: masters restarted or re-elected report no +// filers until the existing ones reconnect. Peers found late are backfilled +// from before their streams start, then reported so the caller waits for +// their catch-up like startup peers. The whole wait — including each peer +// metadata copy — is bounded by the shared announce deadline. +func (fs *FilerServer) awaitPeerBootstrap(deadline time.Time) (peers []pb.ServerAddress) { + confirmBy := time.Now().Add(emptyMembershipConfirmTimeout) + for time.Now().Before(deadline) { + ctx, cancel := context.WithDeadline(context.Background(), deadline) + nodes, listErr := fs.filer.ListExistingPeerUpdates(ctx) + cancel() + if listErr != nil { + glog.Warningf("%s cannot re-check existing peers: %v; announcing anyway", fs.option.Host, listErr) + return nil + } + var remotes []*master_pb.ClusterNodeUpdate + for _, node := range nodes { + if !pb.ServerAddress(node.Address).Equals(fs.option.Host) { + remotes = append(remotes, node) + } + } + if len(remotes) == 0 { + if !time.Now().Before(confirmBy) { + return nil + } + time.Sleep(time.Second) + continue + } + bootstrapFrom := time.Now().Add(-filer.LogFlushInterval) + ctx, cancel = context.WithDeadline(context.Background(), deadline) + err := fs.filer.MaybeBootstrapFromOnePeer(ctx, fs.option.Host, remotes, bootstrapFrom) + cancel() + if err != nil { + glog.Warningf("%s bootstrap from late peers %+v: %v; retrying", fs.option.Host, remotes, err) + time.Sleep(2 * time.Second) + continue + } + glog.V(0).Infof("%s bootstrap from late-arriving peers %+v", fs.option.Host, remotes) + for _, node := range remotes { + peers = append(peers, pb.ServerAddress(node.Address)) + fs.filer.MetaAggregator.OnPeerUpdate(node, bootstrapFrom) + } + return peers + } + return nil +} + // masterRemovalWaitBudget bounds how long shutdown waits for masters to // drop this filer before gRPC drains anyway. const masterRemovalWaitBudget = 5 * time.Second From 2e7e691bed18164de7dcbc8eca9b030647e56674 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 02:30:53 -0700 Subject: [PATCH 02/70] volume: drop stale index entries that resolve to a different needle (#11948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * volume: drop index entries that resolve to a different needle An index row whose offset lands on a needle with a different id and a different size cannot have been produced by any write path: every appender indexes a key at the offset and size of the record it wrote, and WriteNeedleBlob keeps its indexed size consistent with the blob header. Such rows are corruption (for example an offset replayed with stale high bytes) and until now made the volume reject every write for the key with "mismatching cookie" while reads resolved to the wrong needle. When the duplicate check finds the indexed record belongs to a different key and size, drop the stale entry and let the append re-index the key. Same-key conflicts still fail the cookie check, and entries staying consistent with a re-keyed blob keep their existing semantics. * volume: verify the indexed needle header before streaming reads The buffered read path re-parses the record and rejects a needle whose stored size disagrees with the index, but the streamed path serves data bytes without ever looking at the header, so a stale index row would stream another needle's payload. Probe the header once up front and report a miss when the record is neither the requested key nor the indexed size, the same staleness test the write path applies. * rust volume: drop index entries that resolve to a different needle Mirror of the Go volume change: when the duplicate check finds the indexed record belongs to a different key and size, delete the stale entry and let the append re-index the key. * volume: overwrite a stale index entry on commit, not up front Deleting the stale row during the duplicate check published a tombstone before the replacement append and, for durable writes, before its fsync. A failed append or sync then rolled back the data but left the key deleted. The publish step already overwrites any older offset, so leave the entry in place and let it be replaced once the record commits. * volume: report streaming header-probe failures to the io error tracker A failed header read returned before the tracked body-read loop, so repeated EIO on the probe never reached checkReadWriteError. Skip io.EOF the way the loop does: it is a truncated record, not a disk fault. * rust volume: overwrite a stale index entry on commit, not up front Mirror of the Go volume change: the staged delete published a tombstone before the replacement append and its .dat flush, so a failed append or sync left the key deleted. publish_write already overwrites any older offset once the record commits. * rust volume: report needle meta read failures to the io error tracker read_meta is the header probe ahead of a streamed body: an EIO there escaped check_read_write_error entirely, so a failing disk could not trip the tolerance. Report the outcome the way read_full already does. * volume: retry the wrapped offset before judging an index entry stale A 4-byte index value can index past the 32 GiB wrap: the low offset may hold another needle while the real record sits at +MaxPossibleVolumeSize, which is where reads already retry on a size mismatch. Skipping the cookie check on the low-offset record alone would accept a foreign-cookie write over a live key. Retry the wrapped header read first, the same condition the read path uses; a wrapped read that fails for reasons other than EOF stops the write like the low-offset read does. * volume: probe the streamed needle header under the access lock With HasSlowRead the access lock is dropped after the index lookup, so a compaction commit could swap the backend between the lookup and the header probe and fail a valid download on the old offset. Take the read lock for the probe and resolve the compaction revision first, the way the streaming loop re-resolves it on every page. * rust volume: retry the wrapped offset before judging an index entry stale Mirror of the Go volume change: probe offset + MAX_POSSIBLE_VOLUME_SIZE under the 4-byte-offset build before treating a mismatched record as stale, so a live wrapped entry still gets its cookie checked. * rust volume: do not clear the io error streak on a successful meta read read_meta runs before the payload read in read_needle_for_get, so recording success there reset the streak before every failing download and the volume could never reach quarantine. Report failures only; read_full still records success after a complete read. * volume: record a meta-only read's success against the io error streak A meta read no longer clears the streak because a payload read still follows on GETs, but requests the meta read ends outright — HEAD answers and cookie mismatches — never reached the body path, so their success left earlier failures counted. Note the success on those terminal paths; mirrored in the Rust volume server. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * rust volume: report streamed reads to the io error streak Streamed GETs never touched the tracker: a failing chunk read went uncounted and a completed download never cleared the streak, unlike readNeedleDataInto which does both. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: reset the io error streak only at terminal meta reads A streamed GET's metadata read runs before its payload read, so clearing the streak there erased the count before every failing download and the volume could never reach quarantine. Record the meta-read success only where the request actually ends: must-meta-only reads, HEAD responses, and cookie mismatches — matching the Rust volume server's placement. * rust volume: record the streamed read's success at the final frame Hyper can complete a Content-Length response without polling the body again, so a success recorded only when the stream is next polled could be skipped, leaving an earlier io error streak in place after a successful download. * volume: record a meta-only read's success on conditional 304 replies * rust volume: record a meta-only read's success on conditional 304 replies * volume: reset the io error streak only after the checksum passes A streamed read that ends in a CRC mismatch still cleared the EIO streak, so corruption could mask progress toward quarantine. The Rust side records success only after the final frame's checksum verifies; move the reset below the CRC check to match. * volume: satisfy codespell in the meta-read comment * rust volume: report range reads to the io error streak * rust volume: cover the range read's io error reporting --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- seaweed-volume/src/server/handlers.rs | 93 ++++++++++- seaweed-volume/src/storage/volume.rs | 170 +++++++++++++++++++-- weed/server/volume_server_handlers_read.go | 20 +++ weed/storage/volume_idx_repair_test.go | 95 ++++++++++++ weed/storage/volume_read.go | 71 ++++++++- weed/storage/volume_write.go | 44 ++++-- 6 files changed, 465 insertions(+), 28 deletions(-) diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 4eb390f314..6accfed9fe 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -207,9 +207,15 @@ fn read_needle_for_get( if request.try_meta_first(plan.size()) { plan.read_meta(&mut n)?; if n.cookie != cookie { + // The meta read answered the request. + plan.note_meta_only_success(); return Ok(None); } if request.served_from_source(&n) { + if request.is_head { + // HEAD ends here; no payload read will record the success. + plan.note_meta_only_success(); + } let info = plan.into_stream_info(&n); return Ok(Some((n, Some(info)))); } @@ -237,6 +243,9 @@ struct StreamingBody { hold_read_lock_for_stream: bool, _held_read_lease: Option, io_unavailable: Arc, + /// Chunk read failures feed the volume's io-error streak, and finishing + /// the stream clears it — Go readNeedleDataInto parity. + io_errors: Arc, /// Pending chunk read; it hands `buf` back with the result. pending: Option)>>, /// Chunk buffer, reclaimed once the previous frame has been written out. @@ -298,12 +307,18 @@ impl http_body::Body for StreamingBody { ), ))); } + // Hyper may not poll again once Content-Length is + // satisfied, so the read's success is recorded here. + self.io_errors.check_read_write_error(None); } return std::task::Poll::Ready(Some(Ok(http_body::Frame::data( chunk, )))); } - Ok(Err(e)) => return std::task::Poll::Ready(Some(Err(e))), + Ok(Err(e)) => { + self.io_errors.check_read_write_error(Some(&e)); + return std::task::Poll::Ready(Some(Err(e))); + } Err(e) => { return std::task::Poll::Ready(Some(Err(std::io::Error::other(e)))); } @@ -314,6 +329,7 @@ impl http_body::Body for StreamingBody { let total = self.data_size as usize; if self.pos >= total { + self.io_errors.check_read_write_error(None); return std::task::Poll::Ready(None); } @@ -1105,6 +1121,10 @@ async fn get_or_head_handler_inner( let (etag, last_modified_str) = etag_and_last_modified(&n); if let Some(resp) = not_modified_response(&n, &headers, &etag, &last_modified_str) { + // A request that ends here was answered by the meta read alone. + if let Some(io_errors) = strategy.io_errors() { + io_errors.check_read_write_error(None); + } return resp; } @@ -1198,6 +1218,19 @@ struct ReadPlan { strategy: ReadStrategy, } +impl ReadStrategy { + /// The tracker a terminal meta read reports success to, when the reply + /// is served from the data file. + fn io_errors(&self) -> Option<&Arc> { + match self { + ReadStrategy::Stream(info) + | ReadStrategy::HeadFromMeta(info) + | ReadStrategy::RangeFromSource(info) => Some(&info.io_errors), + ReadStrategy::Buffered => None, + } + } +} + /// The 401 reply for a bad read JWT; Go checks it before NewVolumeId, so a /// bad path with JWT enabled is a 401, not a 400. fn reject_read_jwt( @@ -1722,6 +1755,7 @@ fn stream_response( data_file_access_control: info.data_file_access_control, hold_read_lock_for_stream: !state.has_slow_read, io_unavailable: info.io_unavailable, + io_errors: info.io_errors, pending: None, buf: bytes::BytesMut::new(), state: tracking_state, @@ -2121,9 +2155,21 @@ fn handle_range_request_from_source( return Ok(Vec::new()); } let mut buf = vec![0u8; length as usize]; - info.source - .read_exact_at(&mut buf, info.data_file_offset + start as u64)?; - Ok(buf) + match info + .source + .read_exact_at(&mut buf, info.data_file_offset + start as u64) + { + // A finished range read clears the streak a failed meta read + // counted, like readNeedleDataInto's terminal report. + Ok(()) => { + info.io_errors.check_read_write_error(None); + Ok(buf) + } + Err(e) => { + info.io_errors.check_read_write_error(Some(&e)); + Err(e) + } + } }; if ranges.len() == 1 { @@ -5225,6 +5271,7 @@ mod tests { hold_read_lock_for_stream: true, _held_read_lease: None, io_unavailable: Arc::default(), + io_errors: Arc::default(), pending: None, buf: bytes::BytesMut::new(), state: None, @@ -5771,6 +5818,44 @@ mod tests { assert_eq!(status, StatusCode::NOT_FOUND); } + /// A ranged read served from the data file reports its outcome to the + /// io-error streak, like readNeedleDataInto: a finished range clears the + /// count a failed meta read left. + #[test] + fn test_range_from_source_reports_to_io_error_streak() { + use std::io::Write; + + let tmp = tempfile::TempDir::new().unwrap(); + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(tmp.path().join("1.dat")) + .unwrap(); + let data: Vec = (0..256u32).map(|i| i as u8).collect(); + (&file).write_all(&data).unwrap(); + + let io_errors = Arc::new(crate::storage::io_error::IoErrorTracker::default()); + let eio = crate::storage::volume::media_error_for_test(); + io_errors.check_read_write_error(Some(&eio)); + io_errors.check_read_write_error(Some(&eio)); + assert_eq!(io_errors.get_io_error_state().1, 2); + + let info = crate::storage::volume::NeedleStreamInfo { + source: crate::storage::volume::NeedleStreamSource::Local(file), + data_file_offset: 0, + data_size: data.len() as u32, + data_file_access_control: Arc::default(), + io_unavailable: Arc::default(), + io_errors: io_errors.clone(), + needle_id: NeedleId(1), + checksum: 0, + }; + let resp = handle_range_request_from_source("bytes=10-19", info, HeaderMap::new(), None); + assert_eq!(resp.status(), StatusCode::PARTIAL_CONTENT); + assert_eq!(io_errors.get_io_error_state().1, 0); + } + /// Range is read as Go reads it: a byte its parseRange cannot parse is a /// 416, Unicode whitespace around a range is trimmed, an empty value is no /// range, and HEAD ignores it. diff --git a/seaweed-volume/src/storage/volume.rs b/seaweed-volume/src/storage/volume.rs index 83bc73093d..7fc7be57de 100644 --- a/seaweed-volume/src/storage/volume.rs +++ b/seaweed-volume/src/storage/volume.rs @@ -836,7 +836,7 @@ impl NeedleReadPlan { pub(crate) fn read_meta(&mut self, n: &mut Needle) -> Result<(), VolumeError> { let (source, size, version) = (&self.source, self.size, self.version); let read_at = |buf: &mut [u8], at: u64| Ok(source.read_exact_at(buf, at)?); - with_offset_retry(&mut self.offset, |off| { + let result = with_offset_retry(&mut self.offset, |off| { if version == VERSION_1 { // A V1 body is all data, so its "meta" is the whole record. let mut buf = vec![0u8; get_actual_size(size, version) as usize]; @@ -847,13 +847,26 @@ impl NeedleReadPlan { } else { Volume::read_needle_meta_blob_and_parse(read_at, n, off, size, version) } - })?; + }); + // Report failures only: a successful meta read precedes the payload + // read, so clearing the streak here would reset the count before + // every failing download and the volume could never reach quarantine. + if let Err(VolumeError::Io(e)) = &result { + self.io_errors.check_read_write_error(Some(e)); + } + result?; if needle_expired(n) { return Err(VolumeError::NotFound); } Ok(()) } + /// A successful meta read ended the request (e.g. HEAD), so it may reset + /// the streak — unlike a meta read that precedes a payload read. + pub(crate) fn note_meta_only_success(&self) { + self.io_errors.check_read_write_error(None); + } + /// For a reply served from the data file, after `read_meta` filled `n`. pub(crate) fn into_stream_info(self, n: &Needle) -> NeedleStreamInfo { // V1 data starts right after the header, V2/V3 after the DataSize field. @@ -868,6 +881,7 @@ impl NeedleReadPlan { data_size: n.data_size, data_file_access_control: self.data_file_access_control, io_unavailable: self.io_unavailable, + io_errors: self.io_errors, needle_id: self.needle_id, checksum: n.checksum.0, } @@ -1144,6 +1158,9 @@ pub struct NeedleStreamInfo { pub data_file_access_control: Arc, /// Checked before each chunk: the volume can become unavailable mid-stream. pub(crate) io_unavailable: Arc, + /// Streamed chunks report read failures here and clear the streak when + /// the whole body has been emitted, mirroring readNeedleDataInto. + pub(crate) io_errors: Arc, pub needle_id: NeedleId, /// Checksum stored in the needle tail, verified once the last chunk has /// been read — before that frame is emitted. @@ -2918,18 +2935,52 @@ impl Volume { } // Cookie validation for existing needle (matches Go: check whenever nm.Get returns ok) - if let Some(nm) = &io.nm - && let Some(nv) = nm.get(n.id)? + let indexed = match io.nm.as_ref() { + Some(nm) => nm.get(n.id)?, + None => None, + }; + if let Some(nv) = indexed + && !nv.offset.is_zero() { let mut existing = Needle::default(); // Read only the header to check cookie self.read_needle_header_unlocked(io, &mut existing, nv.offset.to_actual_offset())?; - if n.cookie.0 == 0 && !check_cookie { - n.cookie = existing.cookie; + if existing.id != n.id && existing.size != nv.size { + // a 4-byte index entry can index past the 32 GiB wrap: the + // low offset may hold another needle while the real record + // sits at the wrapped offset, where reads already retry + #[cfg(not(feature = "5bytes"))] + if nv.offset.to_actual_offset() < MAX_POSSIBLE_VOLUME_SIZE as i64 { + let mut wrapped = Needle::default(); + match self.read_needle_header_unlocked( + io, + &mut wrapped, + nv.offset.to_actual_offset() + MAX_POSSIBLE_VOLUME_SIZE as i64, + ) { + Ok(()) => existing = wrapped, + Err(VolumeError::Io(e)) if e.kind() == io::ErrorKind::UnexpectedEof => {} + Err(e) => return Err(e), + } + } } - if existing.cookie != n.cookie { - return Err(VolumeError::CookieMismatch(n.cookie.0)); + + if existing.id != n.id && existing.size != nv.size { + // the entry resolves to a record that is neither the indexed + // key nor the indexed size, so it is stale rather than a + // duplicate — skip the cookie check and let publish_write + // overwrite it once the append commits + warn!( + "volume {} index entry for needle {} resolves to needle {}; overwriting stale index entry", + self.id.0, n.id.0, existing.id.0 + ); + } else { + if n.cookie.0 == 0 && !check_cookie { + n.cookie = existing.cookie; + } + if existing.cookie != n.cookie { + return Err(VolumeError::CookieMismatch(n.cookie.0)); + } } } Ok(false) @@ -6067,7 +6118,7 @@ fn preallocate_file(file: &File, size: u64) { /// An OS error the platform reports for failing storage media, which is /// what counts toward the I/O error streak. #[cfg(test)] -fn media_error_for_test() -> io::Error { +pub(crate) fn media_error_for_test() -> io::Error { #[cfg(unix)] { io::Error::from_raw_os_error(libc::EIO) @@ -9044,6 +9095,107 @@ mod tests { assert!(matches!(err, VolumeError::CookieMismatch(_))); } + /// A damaged index entry resolving to a different needle must miss on + /// reads and be dropped by the next write for the key, not rejected as a + /// cookie conflict. + #[test] + fn test_stale_index_entry_heals_on_write() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut first = Needle { + id: NeedleId(2), + cookie: Cookie(0x2222), + data: b"second".to_vec(), + data_size: 6, + ..Needle::default() + }; + v.write_needle(&mut first, true, false).unwrap(); + + // point key 9 at needle 2's record with a size that does not match + // the record, as a corrupt index entry would look + { + let mut io = v.io.write().unwrap(); + let nm = io.nm.as_mut().unwrap(); + let nv = nm.get(NeedleId(2)).unwrap().unwrap(); + nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap(); + } + + let mut probe = Needle { + id: NeedleId(9), + ..Needle::default() + }; + assert!(v.read_needle(&mut probe).is_err()); + + let mut n = Needle { + id: NeedleId(9), + cookie: Cookie(0xabcd), + data: b"ninth".to_vec(), + data_size: 5, + ..Needle::default() + }; + v.write_needle(&mut n, true, false).unwrap(); + + let mut back = Needle { + id: NeedleId(9), + ..Needle::default() + }; + v.read_needle(&mut back).unwrap(); + assert_eq!(back.data, b"ninth"); + assert_eq!(back.cookie, Cookie(0xabcd)); + } + + /// A stale index entry is only replaced once the new record commits: a + /// durable write rolled back by a failed fsync leaves the index exactly + /// as it found it. + #[test] + fn test_stale_index_entry_survives_failed_write() { + let tmp = TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let v = make_test_volume(dir); + + let mut existing = Needle { + id: NeedleId(2), + cookie: Cookie(0x2222), + data: b"second".to_vec(), + data_size: 6, + ..Needle::default() + }; + v.write_needle(&mut existing, true, false).unwrap(); + + let stale_nv = { + let mut io = v.io.write().unwrap(); + let nm = io.nm.as_mut().unwrap(); + let nv = nm.get(NeedleId(2)).unwrap().unwrap(); + nm.put(NeedleId(9), nv.offset, Size(nv.size.0 + 1)).unwrap(); + nm.get(NeedleId(9)).unwrap().unwrap() + }; + + v.fail_next_fsync_for_test(true); + let mut replacement = Needle { + id: NeedleId(9), + cookie: Cookie(0xabcd), + data: b"ninth".to_vec(), + data_size: 5, + ..Needle::default() + }; + assert!(v.write_needle(&mut replacement, true, true).is_err()); + v.fail_next_fsync_for_test(false); + + let now = { + let io = v.io.read().unwrap(); + io.nm + .as_ref() + .unwrap() + .get(NeedleId(9)) + .unwrap() + .expect("a failed write must not drop the index entry it meant to replace") + }; + assert_eq!(now.offset, stale_nv.offset); + assert_eq!(now.size, stale_nv.size); + } + // A size disagreeing with the blob's own header indexes the needle at the wrong // length and, on v3, stamps the append timestamp into the middle of the needle. #[test] diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index aab0245f5e..0c46428920 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -232,6 +232,12 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) return } if n.Cookie != cookie { + // A meta-only read ended the request; record its success. + if readOption.IsMetaOnly { + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } + } glog.V(0).Infof("request %s with cookie:%x expected:%x from %s agent %s", r.URL.Path, cookie, n.Cookie, r.RemoteAddr, r.UserAgent()) NotFound(w) return @@ -241,6 +247,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) if r.Header.Get("If-Modified-Since") != "" { if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { if t.Unix() >= int64(n.LastModified) { + if readOption.IsMetaOnly { + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } + } w.WriteHeader(http.StatusNotModified) return } @@ -248,6 +259,11 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } if inm := r.Header.Get("If-None-Match"); inm == "\""+n.Etag()+"\"" { + if readOption.IsMetaOnly { + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } + } w.WriteHeader(http.StatusNotModified) return } @@ -479,6 +495,10 @@ func (vs *VolumeServer) streamWriteResponseContent(filename string, mimeType str AdjustPassthroughHeaders(w, r, filename) if r.Method == http.MethodHead { + // HEAD ends at the metadata read; record its success. + if v := vs.store.GetVolume(volumeId); v != nil { + v.NoteMetaOnlySuccess() + } w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) return } diff --git a/weed/storage/volume_idx_repair_test.go b/weed/storage/volume_idx_repair_test.go index 197ebfade4..6b155477c5 100644 --- a/weed/storage/volume_idx_repair_test.go +++ b/weed/storage/volume_idx_repair_test.go @@ -1,6 +1,8 @@ package storage import ( + "bytes" + "errors" "os" "path/filepath" "testing" @@ -219,3 +221,96 @@ func TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted(t *testing.T) { t.Fatalf("needle 1 should have been recovered: %v", err) } } + +// TestStaleIndexEntry_HealsOnWrite covers a damaged index entry that resolves +// to a different needle: reads must miss instead of serving the wrong needle, +// and the next write for the key must re-index it rather than be rejected as +// a cookie conflict. +func TestStaleIndexEntry_HealsOnWrite(t *testing.T) { + dir := t.TempDir() + written := writeTestVolume(t, dir, 5) + + v := mustReload(t, dir) + defer v.Close() + + // point key 9 at needle 2's record with a size that does not match the + // record, as a corrupt index entry would look + const staleKey = 9 + nv, ok := v.nm.Get(types.Uint64ToNeedleId(2)) + if !ok { + t.Fatal("needle 2 missing from index") + } + if err := v.nm.Put(types.Uint64ToNeedleId(staleKey), nv.Offset, nv.Size+1); err != nil { + t.Fatalf("plant stale index entry: %v", err) + } + + if _, err := v.readNeedle(newEmptyNeedle(staleKey), nil, nil); err == nil { + t.Fatal("stale-index read served needle 2's data") + } + + stream := newEmptyNeedle(staleKey) + stream.DataSize = written[2].DataSize + if err := v.readNeedleDataInto(stream, &ReadOption{ReadBufferSize: 1024}, &bytes.Buffer{}, 0, int64(stream.DataSize)); err != ErrorNotFound { + t.Fatalf("stale-index stream: want ErrorNotFound, got %v", err) + } + + n := newRandomNeedle(staleKey) + n.Cookie = 0xABCD + if _, _, _, err := v.writeNeedle2(n, true, false, false); err != nil { + t.Fatalf("write over stale index entry: %v", err) + } + if got := mustRead(t, v, staleKey); !bytes.Equal(got, n.Data) { + t.Fatalf("needle %d after re-index: got %d bytes, want %d", staleKey, len(got), len(n.Data)) + } +} + +// TestStaleIndexEntry_SurvivesFailedWrite checks that a stale index entry is +// only replaced once the new record commits: a durable write rolled back by +// a failed fsync leaves the index exactly as it found it. +func TestStaleIndexEntry_SurvivesFailedWrite(t *testing.T) { + v, counting := newCountingVolume(t) + + existing := fixedNeedle(2, "existing") + if _, _, _, err := v.writeNeedle2(existing, true, true, true); err != nil { + t.Fatalf("write needle 2: %v", err) + } + nv, ok := v.nm.Get(existing.Id) + if !ok { + t.Fatal("needle 2 missing from index") + } + + staleId := types.Uint64ToNeedleId(9) + if err := v.nm.Put(staleId, nv.Offset, nv.Size+1); err != nil { + t.Fatalf("plant stale index entry: %v", err) + } + + counting.syncErr = errors.New("batch fsync failed") + if _, _, _, err := v.writeNeedle2(fixedNeedle(9, "replacement"), true, true, false); err == nil { + t.Fatal("a write whose fsync failed must not be acknowledged") + } + + entry, ok := v.nm.Get(staleId) + if !ok { + t.Fatal("a failed write dropped the index entry it meant to replace") + } + if entry.Offset != nv.Offset || entry.Size != nv.Size+1 { + t.Fatalf("index entry changed under a failed write: %+v, want %+v", entry, nv) + } +} + +// TestStaleIndexEntry_KeepsCookieConflictForSameKey checks that an entry +// resolving to the same needle id still enforces the cookie check: only +// cross-key corruption is dropped. +func TestStaleIndexEntry_KeepsCookieConflictForSameKey(t *testing.T) { + dir := t.TempDir() + writeTestVolume(t, dir, 3) + + v := mustReload(t, dir) + defer v.Close() + + n := newRandomNeedle(2) + n.Cookie = 0xABCD + if _, _, _, err := v.writeNeedle2(n, true, false, false); err == nil { + t.Fatal("write with a foreign cookie over an existing key should be rejected") + } +} diff --git a/weed/storage/volume_read.go b/weed/storage/volume_read.go index f93c1a570e..3112cabb15 100644 --- a/weed/storage/volume_read.go +++ b/weed/storage/volume_read.go @@ -61,10 +61,17 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize), readSize, v.Version()) } if err != nil { + v.checkReadWriteError(err) return 0, err } if !n.IsCompressed() && !n.IsChunkedManifest() { readOption.IsMetaOnly = true + // A must-meta-only read ends here, so its success may reset the + // streak. For a streamed GET the meta read precedes the payload read, + // which reports the outcome itself. + if readOption.MustMetaOnly { + v.checkReadWriteError(nil) + } } } if readOption == nil || !readOption.IsMetaOnly { @@ -91,6 +98,13 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize return -1, ErrorNotFound } +// NoteMetaOnlySuccess records a request that ended at the metadata read +// (e.g. a HEAD or a cookie mismatch), so its success may reset the streak — +// unlike a meta read that precedes a payload read. +func (v *Volume) NoteMetaOnlySuccess() { + v.checkReadWriteError(nil) +} + // read needle at a specific offset func (v *Volume) readNeedleMetaAt(n *needle.Needle, offset int64, size int32) (err error) { v.dataFileAccessLock.RLock() @@ -176,6 +190,55 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr } }() + // The header is not part of the streamed body, so verify the indexed + // offset really holds this needle before sending another needle's data. + // A slow read dropped the access lock after the index lookup; take it + // back for the probe so a compaction commit cannot swap the backend + // under it, and resolve the offset the way the streaming loop does. + probeHeader := func() error { + if readOption.VolumeRevision != v.SuperBlock.CompactionRevision { + if v.nm == nil { + return ErrorNotFound + } + refreshed, found := v.nm.Get(n.Id) + if !found || refreshed.Offset.IsZero() { + return ErrorNotFound + } + nv = refreshed + actualOffset = nv.Offset.ToActualOffset() + readOption.VolumeRevision = v.SuperBlock.CompactionRevision + } + header := make([]byte, NeedleHeaderSize) + headerCount, headerErr := v.DataBackend.ReadAt(header, actualOffset) + if headerErr == io.EOF && headerCount == NeedleHeaderSize { + headerErr = nil + } + if headerErr != nil { + if headerErr != io.EOF { + v.checkReadWriteError(headerErr) + } + return fmt.Errorf("ReadNeedleData header: %w", headerErr) + } + probe := new(needle.Needle) + probe.ParseNeedleHeader(header) + if probe.Id != n.Id && probe.Size != nv.Size { + glog.V(0).Infof("volume %d index entry for needle %x resolves to needle %x; reporting not-found", + v.Id, uint64(n.Id), uint64(probe.Id)) + return ErrorNotFound + } + return nil + } + if readOption.HasSlowRead { + v.dataFileAccessLock.RLock() + } + probeErr := probeHeader() + if readOption.HasSlowRead { + v.dataFileAccessLock.RUnlock() + } + if probeErr != nil { + return probeErr + } + // read needle data crc := needle.CRC(0) var pending []byte @@ -255,11 +318,6 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr break } } - // Whole-needle read completed without a backend error — clear any - // pending EIO streak. If a non-EIO failure happens later (CRC etc.) - // we still return that error to the caller, but the disk itself - // produced clean bytes. - v.checkReadWriteError(nil) if checkCRC && (n.Checksum != crc && uint32(n.Checksum) != crc.Value()) { // the crc.Value() function is to be deprecated. this double checking is for backward compatibility // with seaweed version using crc.Value() instead of uint32(crc), which appears in commit 056c480eb @@ -268,6 +326,9 @@ func (v *Volume) readNeedleDataInto(n *needle.Needle, readOption *ReadOption, wr stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorCRC).Inc() return fmt.Errorf("ReadNeedleData checksum %v expected %v for Needle: %v,%v", crc, n.Checksum, v.Id, n) } + // The read succeeded only once the checksum passes; a failed needle + // leaves the io error streak untouched, matching the streamed path. + v.checkReadWriteError(nil) if pending != nil { if _, err = writer.Write(pending); err != nil { return fmt.Errorf("ReadNeedleData write: %w", err) diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index a04fa746cc..88072c9aa3 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -4,6 +4,7 @@ import ( "bytes" "errors" "fmt" + "io" "os" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -322,22 +323,45 @@ func (v *Volume) appendWriteRequest(n *needle.Needle, checkCookie bool, fsync bo // check whether existing needle cookie matches nv, ok := v.nm.Get(n.Id) - if ok { + if ok && !nv.Offset.IsZero() { existingNeedle, _, _, existingNeedleReadErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), nv.Offset.ToActualOffset()) if existingNeedleReadErr != nil { err = fmt.Errorf("reading existing needle: %w", existingNeedleReadErr) return } - if n.Cookie == 0 && !checkCookie { - // this is from batch deletion, and read back again when tailing a remote volume - // which only happens when checkCookie == false and fsync == false - n.Cookie = existingNeedle.Cookie + if existingNeedle.Id != n.Id && existingNeedle.Size != nv.Size && + OffsetSize == 4 && nv.Offset.ToActualOffset() < int64(MaxPossibleVolumeSize) { + // a 4-byte index entry can index past the 32 GiB wrap: the low + // offset may hold another needle while the real record sits at + // the wrapped offset, where reads already retry on a mismatch + wrapped, _, _, wrappedErr := needle.ReadNeedleHeader(v.DataBackend, v.Version(), + nv.Offset.ToActualOffset()+int64(MaxPossibleVolumeSize)) + if wrappedErr == nil { + existingNeedle = wrapped + } else if wrappedErr != io.EOF { + err = fmt.Errorf("reading existing needle at wrapped offset: %w", wrappedErr) + return + } } - if existingNeedle.Cookie != n.Cookie { - glog.V(0).Infof("write cookie mismatch: existing %s, new %s", - needle.NewFileIdFromNeedle(v.Id, existingNeedle), needle.NewFileIdFromNeedle(v.Id, n)) - err = fmt.Errorf("mismatching cookie %x", n.Cookie) - return + if existingNeedle.Id != n.Id && existingNeedle.Size != nv.Size { + // the entry resolves to a record that is neither the indexed key + // nor the indexed size, so it is stale rather than a duplicate — + // skip the cookie check and let the publish step overwrite it + // once the append commits + glog.V(0).Infof("volume %d index entry for %s resolves to %s; overwriting stale index entry", + v.Id, needle.NewFileIdFromNeedle(v.Id, n), needle.NewFileIdFromNeedle(v.Id, existingNeedle)) + } else { + if n.Cookie == 0 && !checkCookie { + // this is from batch deletion, and read back again when tailing a remote volume + // which only happens when checkCookie == false and fsync == false + n.Cookie = existingNeedle.Cookie + } + if existingNeedle.Cookie != n.Cookie { + glog.V(0).Infof("write cookie mismatch: existing %s, new %s", + needle.NewFileIdFromNeedle(v.Id, existingNeedle), needle.NewFileIdFromNeedle(v.Id, n)) + err = fmt.Errorf("mismatching cookie %x", n.Cookie) + return + } } } From bf3081e23a85e660ab889d5fff3cbe61287dafd7 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 03:19:41 -0700 Subject: [PATCH 03/70] s3 lifecycle: route version-file writes to current-version expiration (#11956) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * s3lifecycle: schedule ExpirationDays from version-file writes On a versioned bucket a PUT lands in the meta-log as the version file plus the .versions/ directory update — there is no bare-key event, so the version-folder skip dropped every version-file event for all kinds but ExpiredDeleteMarker. ExpirationDays therefore only fired via the recovery walker's cold-start / rule-change pass: once a shard had a saved cursor, no daily run ever scheduled another delete marker. A version-file write is the latest for its logical key at write time, so route it through the same event-driven evaluation the regular path runs. The match carries the file's identity with an empty VersionID — the dispatcher resolves the current latest at due time, and a fresher PUT meanwhile resolves to STALE_IDENTITY there. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * lifecycle test: versioned Expiration fires on the second daily run Establishes the shard cursor with a first pass, then PUTs and backdates a new versioned object and runs again. The delete marker must appear without a rule change forcing a recovery walk. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: run TestLifecycleVersionedBucketExpiresOnSecondRun in the lifecycle matrix --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/s3-go-tests.yml | 1 + .../lifecycle/s3_lifecycle_versioning_test.go | 49 ++++++++++++ weed/s3api/s3lifecycle/router/router.go | 79 ++++++++++++++++++- weed/s3api/s3lifecycle/router/router_test.go | 77 +++++++++++++++--- 4 files changed, 194 insertions(+), 12 deletions(-) diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 19944e9eb1..3a5c07395b 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -283,6 +283,7 @@ jobs: - TestLifecycleSuspendedVersioningExpiration - TestLifecycleTagFilter - TestLifecycleVersionedBucketCreatesDeleteMarker + - TestLifecycleVersionedBucketExpiresOnSecondRun steps: - name: Check out code diff --git a/test/s3/lifecycle/s3_lifecycle_versioning_test.go b/test/s3/lifecycle/s3_lifecycle_versioning_test.go index c21a37e04b..a25fabcdc3 100644 --- a/test/s3/lifecycle/s3_lifecycle_versioning_test.go +++ b/test/s3/lifecycle/s3_lifecycle_versioning_test.go @@ -164,6 +164,55 @@ func TestLifecycleVersionedBucketCreatesDeleteMarker(t *testing.T) { require.True(t, sawMarker, "ListObjectVersions must show a delete marker as latest for %s", key) } +// TestLifecycleVersionedBucketExpiresOnSecondRun covers the reported +// regression where Expiration.Days only fired on the cold-start walk: a +// versioned PUT surfaces in the meta-log as a version-file event, which +// the router used to drop, so once every shard had a saved cursor no +// daily run ever scheduled another delete marker. The first run here +// establishes the cursor; the second must expire an object that arrived +// in between, with no rule change to force a recovery walk. +func TestLifecycleVersionedBucketExpiresOnSecondRun(t *testing.T) { + c := s3Client(t) + fc, fcClose := filerClient(t) + defer fcClose() + + bucket := uniqueBucket("vexp2") + mustCreateBucket(t, c, bucket) + putVersioningEnabled(t, c, bucket) + putExpirationLifecycle(t, c, bucket, "expire/", 1) + + // First pass: any object will do — what matters is the cursor saved + // afterward, which switches later passes from the recovery walk to + // meta-log replay. + putObject(t, c, bucket, "expire/first.txt", "v1") + runLifecycleShard(t) + + const key = "expire/daily.txt" + putObject(t, c, bucket, key, "v1") + headOut, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + require.NoError(t, err) + versionID := aws.ToString(headOut.VersionId) + require.NotEmpty(t, versionID) + backdateVersionedMtime(t, fc, bucket, key, versionID, 30) + + runLifecycleShard(t) + + require.Eventuallyf(t, func() bool { + _, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + return isS3NotFound(err) + }, 30*time.Second, 500*time.Millisecond, "second run must create a delete marker for %s/%s", bucket, key) + + directHead, err := c.HeadObject(context.Background(), &s3.HeadObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), VersionId: aws.String(versionID), + }) + require.NoError(t, err, "original version must still exist") + require.NotNil(t, directHead) +} + // TestLifecycleNoncurrentVersionExpiration: NoncurrentVersionExpiration // only fires on noncurrent versions. PUT v1, PUT v2 (so v1 → noncurrent), // backdate v1, run worker. v1 must be removed; v2 stays current. diff --git a/weed/s3api/s3lifecycle/router/router.go b/weed/s3api/s3lifecycle/router/router.go index 369172c9c7..81c084e5c1 100644 --- a/weed/s3api/s3lifecycle/router/router.go +++ b/weed/s3api/s3lifecycle/router/router.go @@ -109,11 +109,19 @@ func Route(ctx context.Context, snap *engine.Snapshot, ev *reader.Event, now tim // EXP_DM can fire on two version-folder events: the marker create // (sole survivor immediately) and a noncurrent hard-delete that // leaves only the marker behind. Both reach routeSoleSurvivorMarker. + // A stamped version-file write is also the latest for its logical key + // at write time — the versioned layout emits no bare-key event the + // regular path could see, so ExpirationDays must schedule here or it + // only ever fires via a cold-start / rule-change recovery walk. if versioned && isVersionFolderPath(ev.Key) { - if !hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) { - return nil + var matches []Match + if isVersionFileWrite(ev.NewEntry) { + matches = routeVersionFileExpiration(snap, ev, keys) } - return routeSoleSurvivorMarker(ctx, snap, ev, keys, lister) + if hasActiveEventDrivenAction(snap, keys, s3lifecycle.ActionKindExpiredDeleteMarker) { + matches = append(matches, routeSoleSurvivorMarker(ctx, snap, ev, keys, lister)...) + } + return matches } if ev.NewEntry == nil { @@ -517,6 +525,71 @@ func emitNoncurrentMatches(snap *engine.Snapshot, ev *reader.Event, keys []s3lif return matches } +// isVersionFileWrite reports whether the event's new entry is a genuine +// version-file write: a non-marker file carrying ExtVersionIdKey. The +// version id stamp is what separates it from a literal user file under a +// *.versions/ parent. +func isVersionFileWrite(entry *filer_pb.Entry) bool { + if entry == nil || entry.IsDirectory || isDeleteMarkerEntry(entry) { + return false + } + return len(entry.Extended[s3_constants.ExtVersionIdKey]) > 0 +} + +// routeVersionFileExpiration schedules current-version expiration from a +// version-file write. The file is the latest at write time, and VersionID +// stays empty so the dispatcher resolves the latest at due time — a +// fresher PUT meanwhile resolves to STALE_IDENTITY there, and that PUT's +// own event carried its own schedule. A write that turns out to be an +// older version's metadata update no-ops the same way. +func routeVersionFileExpiration(snap *engine.Snapshot, ev *reader.Event, keys []s3lifecycle.ActionKey) []Match { + logical, ok := logicalKeyFromVersionPath(ev.Key) + if !ok { + return nil + } + entry := ev.NewEntry + if entry.Attributes == nil { + return nil + } + info := &s3lifecycle.ObjectInfo{ + Key: logical, + ModTime: time.Unix(entry.Attributes.Mtime, int64(entry.Attributes.MtimeNs)), + Size: int64(entry.Attributes.FileSize), + IsLatest: true, + } + if tags := extractTags(entry.Extended); len(tags) > 0 { + info.Tags = tags + } + eventTime := time.Unix(0, ev.TsNs) + identity := buildIdentityFromEntry(entry) + var matches []Match + for _, key := range keys { + action := snap.Action(key) + if action == nil || !action.IsActive() || action.Mode != engine.ModeEventDriven { + continue + } + dueTime := s3lifecycle.ComputeDueAt(action.Rule, key.ActionKind, info) + if dueTime.IsZero() { + continue + } + res := s3lifecycle.EvaluateAction(action.Rule, key.ActionKind, info, dueTime) + if res.Action == s3lifecycle.ActionNone { + continue + } + matches = append(matches, Match{ + Key: key, + Action: action, + Result: res, + EventTs: eventTime, + DueTime: dueTime, + Bucket: ev.Bucket, + ObjectKey: logical, + Identity: identity, + }) + } + return matches +} + // routeSoleSurvivorMarker emits an EXP_DM Match against the LOGICAL key // (so the dispatcher can call deleteSpecificObjectVersion) with the // marker's version_id. Handles two events: a marker create (the new diff --git a/weed/s3api/s3lifecycle/router/router_test.go b/weed/s3api/s3lifecycle/router/router_test.go index ddce9a6744..076b0eefb3 100644 --- a/weed/s3api/s3lifecycle/router/router_test.go +++ b/weed/s3api/s3lifecycle/router/router_test.go @@ -458,6 +458,65 @@ func TestRouteVersionedCurrentEventStaysLatest(t *testing.T) { } } +func TestRouteVersionedVersionFileWriteSchedulesExpiration(t *testing.T) { + // A versioned PUT shows up in the meta-log as the version file plus + // the .versions/ directory update — there is no bare-key event for + // the regular path to see. The version file is the latest at write + // time; the match carries its identity with an empty VersionID so + // dispatch CAS-checks the current latest at due time. + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + snap := compileWithVersioned(rule, activatedPrior(rule)) + + now := time.Now() + old := now.AddDate(0, 0, -2) + ev := eventCreate("bk", "logs/foo"+s3_constants.VersionsFolder+"/v_v1", old.Unix(), 1, old.UnixNano()) + ev.NewEntry.Extended = map[string][]byte{ + s3_constants.ExtVersionIdKey: []byte("v1"), + } + + matches := Route(context.Background(), snap, ev, now, nil) + if len(matches) != 1 { + t.Fatalf("expected 1 match (EXPIRATION_DAYS), got %v", matches) + } + m := matches[0] + if m.Key.ActionKind != s3lifecycle.ActionKindExpirationDays { + t.Fatalf("ActionKind=%v, want ExpirationDays", m.Key.ActionKind) + } + if m.ObjectKey != "logs/foo" { + t.Fatalf("ObjectKey=%q, want the logical key", m.ObjectKey) + } + if m.VersionID != "" { + t.Fatalf("VersionID=%q, want empty so dispatch resolves the current latest", m.VersionID) + } + if m.Identity == nil { + t.Fatal("Identity must fingerprint the version file for dispatch CAS") + } + if want := time.Unix(old.Unix(), 0).Add(s3lifecycle.DaysToDuration(1)); !m.DueTime.Equal(want) { + t.Fatalf("DueTime=%v, want %v", m.DueTime, want) + } +} + +func TestRouteVersionedMarkerCreateDoesNotFireExpiration(t *testing.T) { + // A delete marker is a version file too, but ExpirationDays never + // applies to a marker — only EXP_DM, handled by the sole-survivor + // path. Without the marker gate, the expiration match would pin a + // marker identity and dispatch would stack a second marker on top. + rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} + snap := compileWithVersioned(rule, activatedPrior(rule)) + + now := time.Now() + old := now.AddDate(0, 0, -2) + ev := markerEvent("bk", "logs/gone", "2026-05-09-abc", old.Unix(), old.UnixNano()) + + lister := &recordingLister{survivors: Survivors{Count: 1}} + if got := Route(context.Background(), snap, ev, now, lister); len(got) != 0 { + t.Fatalf("marker create must not fire EXPIRATION_DAYS, got %v", got) + } + if len(lister.calls) != 0 { + t.Fatalf("lister consulted without an EXP_DM rule: calls=%v", lister.calls) + } +} + func TestRouteNonVersionedBucketIgnoresVersionsSuffix(t *testing.T) { // A non-versioned bucket happens to have an object literally named // "logs/foo.versions/v1" — that's just a regular path. The router @@ -796,13 +855,11 @@ func TestRouteVersionedDeleteMarkerNoExpDMRuleSkipsListing(t *testing.T) { } func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) { - // On a versioned bucket the router skips every event whose parent - // directory name ends with ".versions" — both the version files - // SeaweedFS itself writes (logs/foo.versions/v_v1) and any literal - // key the user happens to put under such a parent — because the - // current-vs-noncurrent classification needs the .versions/ - // directory's latest pointer, which isn't carried by these events. - // Bootstrap covers retention for those entries. + // On a versioned bucket, events inside a *.versions/ parent that are + // not stamped version-file writes stay unrouted — current-vs-noncurrent + // classification needs the container's latest pointer, and the file's + // own ExtVersionIdKey is what marks it as a version write the + // expiration path can schedule from. rule := &s3lifecycle.Rule{ID: "r", Status: s3lifecycle.StatusEnabled, ExpirationDays: 1} snap := compileWithVersioned(rule, activatedPrior(rule)) @@ -815,9 +872,11 @@ func TestRouteVersionedAllVersionFolderPathsSkipped(t *testing.T) { ext map[string][]byte }{ { - name: "tracked version file", + // A marker carries the version id but is a marker, not an + // expirable write — it routes to the sole-survivor path. + name: "delete marker", key: "logs/foo.versions/v_v1", - ext: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1")}, + ext: map[string][]byte{s3_constants.ExtVersionIdKey: []byte("v1"), s3_constants.ExtDeleteMarkerKey: []byte("true")}, }, { name: "literal-key collision", From ad89f003a53187ec82975e077306110bb9f57865 Mon Sep 17 00:00:00 2001 From: Javier de la Torre Date: Fri, 23 Oct 2026 12:58:06 +0200 Subject: [PATCH 04/70] s3: end a catalog-vended session with the access key it was vended for (#11945) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * s3: end a catalog-vended session with the access key it was vended for A table credential the Iceberg catalog vends is a self-contained STS session: it stays valid until it expires (an hour by default) even after the access key that loaded the table is deleted or deactivated, or its identity is removed or disabled. An operator revoking a leaked key cannot cut off the table credentials that key already obtained. Sessions are stateless JWTs, so there is no revocation list to consult. This change follows the same pattern as role sessions, which are already refused once their role's ID no longer matches (checkSessionRoleBinding): a vended session carries the access key it was vended for (src_ak, recorded by the previous change), and validateSTSSessionToken checks it against the identity table held in memory. A session of provider iceberg-catalog that carries a source key is refused (InvalidAccessKeyId, as for an unknown key) once its identity is gone or disabled, or the key is no longer one of that identity's active, unexpired credentials. Scope: - Only catalog-vended sessions are checked; web-identity, AssumeRole and every other STS session are validated as before. - The vendor binds a session to the caller's key only when the caller is a stored identity holding that key. A caller that signed with another STS session's key, an OIDC user, or a request that recorded no key gets an unbound session, as today; binding it would refuse every request made with it. A session minted before this change carries no key and is unbound. - The check is one map lookup under the read lock and a scan of one identity's keys (about 11 ns, 0 allocations, with 1,000 identities). Tests (s3api_iceberg_vended_session_test.go) mint real sessions through the catalog vendor and validate them the way a data-path request is validated: deactivating, expiring or deleting key A ends A's session while the session vended through key B on the same identity keeps working; disabling or deleting the identity ends it; callers that are not a stored identity holding the key get unbound sessions that keep working; other STS sessions are not checked. The first two tests and the third fail without this change. * s3: keep the vended-session key binding when the key dies before vend vendedSessionKeyLive gated the binding, so a key revoked between the request's authentication and the vend skipped it and minted an unbound session that stayed usable until it expired. Every catalog auth path pairs the identity name in context with the key that authenticated, so a stored-identity principal always means the request signed with that identity's own key. Bind whenever the principal is a stored identity: a session bound to a now-dead key is refused on its first request, which is what the binding is for. * s3: gate the vended-session key binding on the authenticated caller type lookupByIdentityName(principal) named only that some identity exists, not that the request signed with a stored key: an STS session or OIDC principal that collides with an identity name got a session bound to its temporary key, which vendedSessionKeyLive then refused on every request. Verifiers now record the verified key as a stored credential, and the vendor binds only when that marker is present — keeping the binding even when the key was revoked between authentication and the vend. * s3: enforce the vended key binding under a dedicated claim Sessions minted before the binding existed carry the caller's key in the audit-only SourceAccessKeyId — including temporary STS keys that can never resolve in the identity table. Enforcing on that field would end live table credentials at upgrade; the binding now travels under BoundAccessKeyId, leaving legacy sessions unbound as before. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: bind vended sessions to a credential fingerprint The key id alone cannot identify a credential: an operator can delete a key and re-create it under the same id with a different secret, reviving sessions minted for the original. Verifiers now record a fingerprint of the secret at authentication; the session dies unless it still matches. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: keep recording the caller's key on every catalog session SourceAccessKeyId was unconditional audit metadata before the binding existed; gating it on the stored-credential marker dropped the record for callers that authenticated another way. With the binding under its own claim there is no ambiguity left, so the field records the caller's key again regardless of how it authenticated. * s3: key the vended-session credential fingerprint with the signing key The bound-credential fingerprint travels inside the vended session token. As a plain SHA-256 of the secret it gave any token holder an offline guessing oracle for weak secrets, and CodeQL flags it twice. Store the verified secret in the request marker instead and HMAC it with the STS signing key at vend and at validation. The claim can no longer be mined, and the per-request auth path stops hashing secrets. --------- Co-authored-by: Javier de la Torre Co-authored-by: Chris Lu Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/iam/sts/session_claims.go | 18 ++ weed/iam/sts/sts_service.go | 23 +- weed/s3api/auth_signature_v2.go | 3 + weed/s3api/auth_signature_v4.go | 49 ++++ weed/s3api/iceberg/handlers_oauth.go | 22 +- weed/s3api/iceberg/server.go | 3 +- weed/s3api/lance/handlers_oauth.go | 22 +- weed/s3api/lance/server.go | 6 +- weed/s3api/s3_constants/header.go | 103 +++++++- weed/s3api/s3_constants/header_test.go | 17 ++ weed/s3api/s3api_iceberg_credentials.go | 38 ++- weed/s3api/s3api_iceberg_session_name_test.go | 4 +- .../s3api_iceberg_vended_session_test.go | 234 ++++++++++++++++++ 13 files changed, 507 insertions(+), 35 deletions(-) create mode 100644 weed/s3api/s3api_iceberg_vended_session_test.go diff --git a/weed/iam/sts/session_claims.go b/weed/iam/sts/session_claims.go index e7f4ce89c5..0dd0361718 100644 --- a/weed/iam/sts/session_claims.go +++ b/weed/iam/sts/session_claims.go @@ -105,6 +105,13 @@ type STSSessionClaims struct { // a SeaweedFS service minted the session for it (AssumeRoleForPrincipal); // empty for every other session type. SourceAccessKeyId string `json:"src_ak,omitempty"` + + // BoundAccessKeyId is the stored credential the session is bound to: the + // session ends when that key is gone. It is a claim of its own because + // tokens minted before the binding existed carry SourceAccessKeyId as an + // audit field, never a binding — enforcing those would break them. + BoundAccessKeyId string `json:"bind_ak,omitempty"` + BoundAccessKeyFingerprint string `json:"bind_fp,omitempty"` } // NewSTSSessionClaims creates new STS session claims with all required information @@ -163,6 +170,9 @@ func (c *STSSessionClaims) ToSessionInfo(credGen *CredentialGenerator) *SessionI ParentUser: c.ParentUser, // The access key the session was minted for, if recorded SourceAccessKeyId: c.SourceAccessKeyId, + // The stored credential the session is bound to, if any + BoundAccessKeyId: c.BoundAccessKeyId, + BoundAccessKeyFingerprint: c.BoundAccessKeyFingerprint, // Provide the Subject (sub) from registered claims Subject: c.Subject, Credentials: credentials, @@ -263,6 +273,14 @@ func (c *STSSessionClaims) WithSourceAccessKeyId(accessKeyId string) *STSSession return c } +// WithBoundAccessKey binds the session to a stored credential: the key id it +// was vended for and a fingerprint of its secret at mint time. +func (c *STSSessionClaims) WithBoundAccessKey(accessKeyId, fingerprint string) *STSSessionClaims { + c.BoundAccessKeyId = accessKeyId + c.BoundAccessKeyFingerprint = fingerprint + return c +} + // WithParentUser sets the stable per-identity hash for the session. See // ComputeParentUser for the derivation rule. func (c *STSSessionClaims) WithParentUser(parentUser string) *STSSessionClaims { diff --git a/weed/iam/sts/sts_service.go b/weed/iam/sts/sts_service.go index 9083b221e5..99f48dc14d 100644 --- a/weed/iam/sts/sts_service.go +++ b/weed/iam/sts/sts_service.go @@ -320,6 +320,16 @@ type SessionInfo struct { // SourceAccessKeyId is the access key the session was minted for, when the // minting service recorded it (AssumeRoleForPrincipal). SourceAccessKeyId string `json:"sourceAccessKeyId,omitempty"` + + // BoundAccessKeyId is the stored credential the session is bound to, when + // the minting service recorded one (AssumeRoleForPrincipal). A legacy + // SourceAccessKeyId is never treated as a binding. + BoundAccessKeyId string `json:"boundAccessKeyId,omitempty"` + + // BoundAccessKeyFingerprint identifies the credential instance the + // session is bound to, so a key re-created under the same id does not + // match. + BoundAccessKeyFingerprint string `json:"boundAccessKeyFingerprint,omitempty"` } // NewSTSService creates a new STS service @@ -901,6 +911,14 @@ type AssumeRoleForPrincipalRequest struct { // SourceAccessKeyId optionally records the access key the caller // authenticated with, so the session can be tied to that key. SourceAccessKeyId string + + // BoundAccessKeyId binds the session to a stored credential: the session + // ends when that key is gone. Distinct from SourceAccessKeyId, which is + // audit-only on sessions minted before the binding existed. + BoundAccessKeyId string + + // BoundAccessKeyFingerprint identifies the bound credential instance. + BoundAccessKeyFingerprint string } // AssumeRoleForPrincipal issues session credentials for a caller that a @@ -947,7 +965,10 @@ func (s *STSService) AssumeRoleForPrincipal(ctx context.Context, request *Assume // carry no role ID and stay bound by name. return s.issueSession(request.RoleArn, "", request.RoleSessionName, sessionPolicy, request.DurationSeconds, request.ProviderName, request.Principal, - func(c *STSSessionClaims) { c.WithSourceAccessKeyId(request.SourceAccessKeyId) }) + func(c *STSSessionClaims) { + c.WithSourceAccessKeyId(request.SourceAccessKeyId) + c.WithBoundAccessKey(request.BoundAccessKeyId, request.BoundAccessKeyFingerprint) + }) } // ValidateSessionToken validates a session token and returns session information diff --git a/weed/s3api/auth_signature_v2.go b/weed/s3api/auth_signature_v2.go index 3bda791e64..5148be2218 100644 --- a/weed/s3api/auth_signature_v2.go +++ b/weed/s3api/auth_signature_v2.go @@ -110,6 +110,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV2Match(ctx context.Cont return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(ctx, cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } @@ -168,6 +169,7 @@ func (iam *IdentityAccessManagement) doesSignV2Match(r *http.Request) (*Identity return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } @@ -227,6 +229,7 @@ func (iam *IdentityAccessManagement) doesPresignV2SignatureMatch(r *http.Request return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/auth_signature_v4.go b/weed/s3api/auth_signature_v4.go index b0ec7fd292..5f96669554 100644 --- a/weed/s3api/auth_signature_v4.go +++ b/weed/s3api/auth_signature_v4.go @@ -412,6 +412,9 @@ func (iam *IdentityAccessManagement) verifyV4Signature(r *http.Request, shouldCh // catalog's vended sessions name it (never a key re-read from the request). // Recorded before the retarget check so that refusal names it too. s3_constants.RecordAccessKeyId(r.Context(), cred.AccessKey) + if found { + s3_constants.RecordStoredAccessKeyId(r.Context(), cred.AccessKey, cred.SecretKey) + } if signedBucket, ok := bucketFromVirtualHost(matchedHost, iam.domain); ok { if routedBucket, _ := s3_constants.GetBucketAndObject(r); routedBucket != "" && routedBucket != signedBucket { glog.V(2).Infof("reject %s %s: signed host %q implies bucket %q but routed to %q", @@ -496,6 +499,19 @@ func (iam *IdentityAccessManagement) validateSTSSessionToken(r *http.Request, se return nil, nil, s3err.ErrExpiredToken } + // A session the table catalog vended for a static access key is bound to + // that key, as a role session is bound to its role's ID: deleting or + // deactivating the key, or deleting or disabling its identity, ends the + // session at its next request instead of at its expiry. Sessions minted + // before the binding existed carry only the audit field and stay unbound. + if sessionInfo.IdentityProvider == icebergVendedCredentialsProvider && sessionInfo.BoundAccessKeyId != "" { + if errCode := iam.vendedSessionKeyLive(sessionInfo.ExternalUserId, sessionInfo.BoundAccessKeyId, sessionInfo.BoundAccessKeyFingerprint); errCode != s3err.ErrNone { + glog.V(2).Infof("vended STS session %s: identity %q or the access key it was vended for is gone, disabled or inactive", + sessionInfo.SessionId, sessionInfo.ExternalUserId) + return nil, nil, errCode + } + } + // Validate required credential fields if sessionInfo.Credentials.SecretAccessKey == "" { glog.Warningf("STS service returned empty SecretAccessKey in credentials") @@ -554,6 +570,38 @@ func (iam *IdentityAccessManagement) validateSTSSessionToken(r *http.Request, se return identity, cred, s3err.ErrNone } +// vendedSessionKeyLive answers ErrNone while owner is a known, enabled +// identity holding accessKey as an active, unexpired credential, and +// ErrInvalidAccessKeyID (as for an unknown key) otherwise. A fingerprint set +// at mint time must still match, so a key re-created under the same id does +// not revive sessions the deleted original had carried. One map lookup under +// the read lock. +func (iam *IdentityAccessManagement) vendedSessionKeyLive(owner, accessKey, fingerprint string) s3err.ErrorCode { + if owner == "" || accessKey == "" { + return s3err.ErrInvalidAccessKeyID + } + fingerprintKey := iam.stsSigningKey() + iam.m.RLock() + defer iam.m.RUnlock() + identity, ok := iam.nameToIdentity[owner] + if !ok || identity == nil || identity.Disabled { + return s3err.ErrInvalidAccessKeyID + } + for _, credential := range identity.Credentials { + if credential == nil || credential.AccessKey != accessKey { + continue + } + if credential.Status == iamAccessKeyStatusInactive || credential.isCredentialExpired() { + return s3err.ErrInvalidAccessKeyID + } + if fingerprint != "" && s3_constants.StoredCredentialFingerprint(fingerprintKey, accessKey, credential.SecretKey) != fingerprint { + return s3err.ErrInvalidAccessKeyID + } + return s3err.ErrNone + } + return s3err.ErrInvalidAccessKeyID +} + // calculateAndVerifySignature contains the core logic for creating the canonical request, // string-to-sign, and comparing the final signature. func calculateAndVerifySignature(secretKey, method, urlPath, queryStr string, extractedSignedHeaders http.Header, authInfo *v4AuthInfo) (string, s3err.ErrorCode) { @@ -874,6 +922,7 @@ func (iam *IdentityAccessManagement) doesPolicySignatureV4Match(ctx context.Cont return nil, s3err.ErrSignatureDoesNotMatch } s3_constants.RecordAccessKeyId(ctx, cred.AccessKey) + s3_constants.RecordStoredAccessKeyId(ctx, cred.AccessKey, cred.SecretKey) return identity, s3err.ErrNone } diff --git a/weed/s3api/iceberg/handlers_oauth.go b/weed/s3api/iceberg/handlers_oauth.go index 8e6933c461..54391d04e4 100644 --- a/weed/s3api/iceberg/handlers_oauth.go +++ b/weed/s3api/iceberg/handlers_oauth.go @@ -324,28 +324,28 @@ func mintIcebergToken(identityName, accessKey, secret string, ttlSeconds int) (s // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { - identityName, identity, _, ok := s.authenticateBearerKey(r) + identityName, identity, _, _, ok := s.authenticateBearerKey(r) return identityName, identity, ok } // authenticateBearerKey is authenticateBearer that also returns the access key // the token was minted from (its verified access_key claim), so the catalog // can tell apart the holders of several keys on one identity. -func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, "", false + return "", nil, "", "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, "", false + return "", nil, "", "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, "", false + return "", nil, "", "", false } if s.credentialValidator == nil { - return "", nil, "", false + return "", nil, "", "", false } // Parse the token without verification first to get the access key, @@ -355,18 +355,18 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to parse token: %v", err) - return "", nil, "", false + return "", nil, "", "", false } if unverified.AccessKey == "" { - return "", nil, "", false + return "", nil, "", "", false } // Look up the credential by access key to get the signing key for verification identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Iceberg OAuth: failed to get credential for access key: %v", err) - return "", nil, "", false + return "", nil, "", "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -379,12 +379,12 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st }) if err != nil || !verified.Valid { glog.V(2).Infof("Iceberg OAuth: token verification failed: %v", err) - return "", nil, "", false + return "", nil, "", "", false } // The token's signing key is derived from this access key, so a verified // token was minted from it. - return identityName, identity, unverified.AccessKey, true + return identityName, identity, unverified.AccessKey, secretKey, true } // deriveSigningKey derives a signing key from the access key and secret using HMAC-SHA256. diff --git a/weed/s3api/iceberg/server.go b/weed/s3api/iceberg/server.go index 5ec8de2160..1849f2e6e8 100644 --- a/weed/s3api/iceberg/server.go +++ b/weed/s3api/iceberg/server.go @@ -269,10 +269,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // The auth scheme is case-insensitive (RFC 7235), matching // authenticateBearer below. if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { + if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } diff --git a/weed/s3api/lance/handlers_oauth.go b/weed/s3api/lance/handlers_oauth.go index 20e791221e..d348a5c200 100644 --- a/weed/s3api/lance/handlers_oauth.go +++ b/weed/s3api/lance/handlers_oauth.go @@ -147,27 +147,27 @@ func mintToken(identityName, accessKey, secret string, ttlSeconds int) (string, // authenticateBearer validates a Bearer token from the Authorization header. // Returns the identity name, identity object, and whether auth succeeded. func (s *Server) authenticateBearer(r *http.Request) (string, interface{}, bool) { - identityName, identity, _, ok := s.authenticateBearerKey(r) + identityName, identity, _, _, ok := s.authenticateBearerKey(r) return identityName, identity, ok } // authenticateBearerKey is authenticateBearer that also returns the access key // the token was minted from (its verified access_key claim). -func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, bool) { +func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, string, string, bool) { auth := r.Header.Get("Authorization") if auth == "" { - return "", nil, "", false + return "", nil, "", "", false } if !strings.HasPrefix(strings.ToLower(auth), "bearer ") { - return "", nil, "", false + return "", nil, "", "", false } tokenString := strings.TrimSpace(auth[7:]) if tokenString == "" { - return "", nil, "", false + return "", nil, "", "", false } if s.credentialValidator == nil { - return "", nil, "", false + return "", nil, "", "", false } // Parse the token without verification first to get the access key, @@ -177,17 +177,17 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st _, _, err := parser.ParseUnverified(tokenString, unverified) if err != nil { glog.V(2).Infof("Lance OAuth: failed to parse token: %v", err) - return "", nil, "", false + return "", nil, "", "", false } if unverified.AccessKey == "" { - return "", nil, "", false + return "", nil, "", "", false } identityName, identity, secretKey, err := s.credentialValidator.GetCredentialByAccessKey(unverified.AccessKey) if err != nil { glog.V(2).Infof("Lance OAuth: failed to get credential for access key: %v", err) - return "", nil, "", false + return "", nil, "", "", false } signingKey := deriveSigningKey(unverified.AccessKey, secretKey) @@ -200,12 +200,12 @@ func (s *Server) authenticateBearerKey(r *http.Request) (string, interface{}, st }) if err != nil || !verified.Valid { glog.V(2).Infof("Lance OAuth: token verification failed: %v", err) - return "", nil, "", false + return "", nil, "", "", false } // The token's signing key is derived from this access key, so a verified // token was minted from it. - return identityName, identity, unverified.AccessKey, true + return identityName, identity, unverified.AccessKey, secretKey, true } // authenticateApiKey validates an x-api-key header carrying an S3 credential diff --git a/weed/s3api/lance/server.go b/weed/s3api/lance/server.go index d0cb813daa..53effee9e5 100644 --- a/weed/s3api/lance/server.go +++ b/weed/s3api/lance/server.go @@ -177,10 +177,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { // different error that clients cannot act on. // The auth scheme is case-insensitive (RFC 7235). if strings.HasPrefix(strings.ToLower(r.Header.Get("Authorization")), "bearer ") { - if identityName, identity, accessKey, ok := s.authenticateBearerKey(r); ok { + if identityName, identity, accessKey, secretKey, ok := s.authenticateBearerKey(r); ok { ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } @@ -195,10 +196,11 @@ func (s *Server) Auth(handler http.HandlerFunc) http.HandlerFunc { if apiKey := r.Header.Get("x-api-key"); apiKey != "" { if identityName, identity, ok := s.authenticateApiKey(apiKey); ok { - accessKey, _, _ := strings.Cut(apiKey, ":") + accessKey, secretKey, _ := strings.Cut(apiKey, ":") ctx := r.Context() ctx = s3_constants.SetIdentityNameInContext(ctx, identityName) ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secretKey) if identity != nil { ctx = s3_constants.SetIdentityInContext(ctx, identity) } diff --git a/weed/s3api/s3_constants/header.go b/weed/s3api/s3_constants/header.go index cf26ac8f84..a49d40220e 100644 --- a/weed/s3api/s3_constants/header.go +++ b/weed/s3api/s3_constants/header.go @@ -18,6 +18,9 @@ package s3_constants import ( "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "net/http" "strings" "sync/atomic" @@ -355,6 +358,7 @@ const ( contextKeyIgnoredSessionToken contextKey = "s3-ignored-session-token" contextKeyOriginalRequestPath contextKey = "s3-original-request-path" contextKeyAccessKeyId contextKey = "s3-access-key-id" + contextKeyStoredAccessKeyId contextKey = "s3-stored-access-key-id" ) // identityHolder is a mutable container for the authenticated identity name, @@ -365,10 +369,37 @@ const ( // holder installed before authentication is shared across all copies, so the // name written by the inner handler is readable by the outer middleware. type identityHolder struct { - name atomic.Pointer[string] - principalArn atomic.Pointer[string] - identityClaim atomic.Pointer[string] - accessKeyId atomic.Pointer[string] + name atomic.Pointer[string] + principalArn atomic.Pointer[string] + identityClaim atomic.Pointer[string] + accessKeyId atomic.Pointer[string] + storedCredential atomic.Pointer[storedCredential] +} + +// storedCredential is the stored-identity credential a request authenticated +// with: the access key id and its secret, so a credential vendor can bind +// what it mints to this exact credential instance. +type storedCredential struct { + accessKeyId string + secretKey string +} + +// StoredCredentialFingerprint identifies one credential instance. The key id +// alone cannot: an operator can delete a key and re-create it under the same +// id with a different secret. The fingerprint travels inside the vended +// session token, so it is keyed with a server-held key — an unkeyed hash of +// the secret would hand the token holder an offline guessing oracle. +func StoredCredentialFingerprint(fingerprintKey []byte, accessKeyId, secretKey string) string { + if len(fingerprintKey) == 0 { + return "" + } + h := hmac.New(sha256.New, fingerprintKey) + h.Write([]byte("seaweedfs-stored-credential")) + h.Write([]byte{0}) + h.Write([]byte(accessKeyId)) + h.Write([]byte{0}) + h.Write([]byte(secretKey)) + return hex.EncodeToString(h.Sum(nil)[:16]) } // EnsureIdentityHolder attaches a mutable identity holder to the request context @@ -554,6 +585,70 @@ func AccessKeyIdFromContext(ctx context.Context) string { return "" } +// SetStoredAccessKeyIdInContext records the stored-identity credential the +// request authenticated with — set by an authenticator that verified the +// credential against the identity store, so a credential vendor can tell the +// caller apart from an STS session or OIDC user holding a key the store never +// issued. +func SetStoredAccessKeyIdInContext(ctx context.Context, accessKeyId, secretKey string) context.Context { + if accessKeyId == "" { + return ctx + } + sc := storedCredential{accessKeyId, secretKey} + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.storedCredential.Store(&sc) + } + return context.WithValue(ctx, contextKeyStoredAccessKeyId, sc) +} + +// RecordStoredAccessKeyId stores the stored-credential marker into the +// request's identity holder: a verifier calls it when the credential that +// matched resolves to a key held by a configured identity. A no-op without a +// holder (EnsureIdentityHolder) or without a key. +func RecordStoredAccessKeyId(ctx context.Context, accessKeyId, secretKey string) { + if ctx == nil || accessKeyId == "" { + return + } + sc := storedCredential{accessKeyId, secretKey} + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + h.storedCredential.Store(&sc) + } +} + +// StoredAccessKeyIdFromContext retrieves the stored-identity access key id the +// request authenticated with, or "" when it authenticated another way (STS +// session, OIDC bearer, anonymous). +func StoredAccessKeyIdFromContext(ctx context.Context) string { + sc, _ := storedCredentialFromContext(ctx) + return sc.accessKeyId +} + +// StoredCredentialFromContext retrieves the stored-identity access key id and +// secret the request authenticated with, or "", "" when it authenticated +// another way (STS session, OIDC bearer, anonymous). +func StoredCredentialFromContext(ctx context.Context) (string, string) { + sc, ok := storedCredentialFromContext(ctx) + if !ok { + return "", "" + } + return sc.accessKeyId, sc.secretKey +} + +func storedCredentialFromContext(ctx context.Context) (storedCredential, bool) { + if ctx == nil { + return storedCredential{}, false + } + if sc, ok := ctx.Value(contextKeyStoredAccessKeyId).(storedCredential); ok && sc.accessKeyId != "" { + return sc, true + } + if h, ok := ctx.Value(contextKeyIdentityHolder).(*identityHolder); ok && h != nil { + if sc := h.storedCredential.Load(); sc != nil && sc.accessKeyId != "" { + return *sc, true + } + } + return storedCredential{}, false +} + // IgnoreSessionTokenInContext marks the request's session token as tolerated // rather than authenticated: a statically configured access key signed the // request, so the token (which credential vendors like Unity Catalog emit on diff --git a/weed/s3api/s3_constants/header_test.go b/weed/s3api/s3_constants/header_test.go index de780d0d51..2d30f3e859 100644 --- a/weed/s3api/s3_constants/header_test.go +++ b/weed/s3api/s3_constants/header_test.go @@ -313,3 +313,20 @@ func TestGetIdentityNameFromContextWithoutHolder(t *testing.T) { t.Errorf("requester = %q, want admin", got) } } + +func TestStoredCredentialFingerprintIsKeyed(t *testing.T) { + key := []byte("test-signing-key-32-characters!!") + fp := StoredCredentialFingerprint(key, "AKIAEXAMPLE", "secret") + if fp == "" { + t.Fatal("fingerprint is empty with a key") + } + if again := StoredCredentialFingerprint(key, "AKIAEXAMPLE", "secret"); again != fp { + t.Error("fingerprint is not deterministic") + } + if other := StoredCredentialFingerprint([]byte("another-signing-key-padded-out"), "AKIAEXAMPLE", "secret"); other == fp { + t.Error("fingerprint ignores the server key, leaving the secret guessable") + } + if noKey := StoredCredentialFingerprint(nil, "AKIAEXAMPLE", "secret"); noKey != "" { + t.Error("an empty key must yield an empty fingerprint") + } +} diff --git a/weed/s3api/s3api_iceberg_credentials.go b/weed/s3api/s3api_iceberg_credentials.go index 50af7cdb99..967a5a79df 100644 --- a/weed/s3api/s3api_iceberg_credentials.go +++ b/weed/s3api/s3api_iceberg_credentials.go @@ -40,8 +40,11 @@ type VendedCredentials struct { // // The access key the caller authenticated with (from ctx) is recorded in the // session: a short hash of it ends the session name, so the audit log's -// requester_arn tells apart the holders of several keys on one identity, and -// the key id itself is carried in the session token. +// requester_arn tells apart the holders of several keys on one identity. When +// the caller is a stored identity holding that key, the key id is also carried +// in the session token, binding the session to it: the session ends when the +// key is deleted or deactivated, or its identity removed or disabled +// (validateSTSSessionToken). func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, bucket, prefix string, readOnly bool) (*VendedCredentials, error) { if s3a.icebergCredentialRole == "" { return nil, nil @@ -72,6 +75,19 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc Policy: &policy, SourceAccessKeyId: accessKey, } + // Bind the session to the caller's key when the caller authenticated with a + // stored identity's key — a marker its verifier records, so a caller that + // signed with an STS session key or OIDC token stays unbound even when its + // principal names a stored identity, whose session a binding would refuse + // on every request. The marker survives the key being revoked between + // authentication and this vend: the session then dies on its first request + // instead of living unbound until it expires. The binding travels under + // its own claim — SourceAccessKeyId is audit-only for compatibility with + // tokens minted before the binding existed. + if storedKey, storedSecret := s3_constants.StoredCredentialFromContext(ctx); storedKey != "" { + request.BoundAccessKeyId = storedKey + request.BoundAccessKeyFingerprint = s3_constants.StoredCredentialFingerprint(s3a.iam.stsSigningKey(), storedKey, storedSecret) + } if s3a.icebergCredentialDuration > 0 { duration := s3a.icebergCredentialDuration request.DurationSeconds = &duration @@ -94,10 +110,17 @@ func (s3a *S3ApiServer) VendTableCredentials(ctx context.Context, principal, buc } func (s3a *S3ApiServer) stsService() *sts.STSService { - if s3a.iam == nil || s3a.iam.iamIntegration == nil { + if s3a.iam == nil { return nil } - provider, ok := s3a.iam.iamIntegration.(IAMManagerProvider) + return s3a.iam.stsService() +} + +func (iam *IdentityAccessManagement) stsService() *sts.STSService { + if iam.iamIntegration == nil { + return nil + } + provider, ok := iam.iamIntegration.(IAMManagerProvider) if !ok { return nil } @@ -108,6 +131,13 @@ func (s3a *S3ApiServer) stsService() *sts.STSService { return manager.GetSTSService() } +func (iam *IdentityAccessManagement) stsSigningKey() []byte { + if s := iam.stsService(); s != nil && s.Config != nil { + return s.Config.SigningKey + } + return nil +} + // tablePrefixSessionPolicy narrows a session to one table's files plus the // bucket listing that clients need to resolve them. A read-only session drops // the write verbs: the catalog vends no more authority than the caller holds. diff --git a/weed/s3api/s3api_iceberg_session_name_test.go b/weed/s3api/s3api_iceberg_session_name_test.go index 9a615bef8d..2fd679d24d 100644 --- a/weed/s3api/s3api_iceberg_session_name_test.go +++ b/weed/s3api/s3api_iceberg_session_name_test.go @@ -76,12 +76,14 @@ func TestVendTableCredentialsRecordsTheCallersKey(t *testing.T) { s3a := &S3ApiServer{iam: iam} s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600) - for _, key := range []string{sharedKeyA, sharedKeyB, ""} { + for _, k := range []struct{ id, secret string }{{sharedKeyA, sharedSecretA}, {sharedKeyB, sharedSecretB}, {"", ""}} { + key := k.id ctx := context.Background() if key != "" { // As the catalog's auth does: the holder is installed, the verifier records into it. r := s3_constants.EnsureIdentityHolder(httptest.NewRequest(http.MethodGet, "/", nil)) s3_constants.RecordAccessKeyId(r.Context(), key) + s3_constants.RecordStoredAccessKeyId(r.Context(), k.id, k.secret) ctx = r.Context() } creds, err := s3a.VendTableCredentials(ctx, "shared-user", "warehouse", "ns/t", false) diff --git a/weed/s3api/s3api_iceberg_vended_session_test.go b/weed/s3api/s3api_iceberg_vended_session_test.go new file mode 100644 index 0000000000..9b9398cf0f --- /dev/null +++ b/weed/s3api/s3api_iceberg_vended_session_test.go @@ -0,0 +1,234 @@ +package s3api + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/seaweedfs/seaweedfs/weed/iam/integration" + "github.com/seaweedfs/seaweedfs/weed/iam/policy" + "github.com/seaweedfs/seaweedfs/weed/iam/sts" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" +) + +// vendingServer is an S3ApiServer whose catalog role trusts the vendor, with +// the identity of newSharedIdentityIAM (one identity, two keys). +func vendingServer(t *testing.T) *S3ApiServer { + t.Helper() + mgr := newTestIAMManager(t) + require.NoError(t, mgr.CreateRole(context.Background(), "", "IcebergTableAccess", &integration.RoleDefinition{ + RoleName: "IcebergTableAccess", + TrustPolicy: &policy.PolicyDocument{ + Version: "2012-10-17", + Statement: []policy.Statement{{ + Effect: "Allow", + Principal: map[string]interface{}{"Federated": icebergVendedCredentialsProvider}, + Action: []string{"sts:AssumeRoleWithCredentials"}, + }}, + }, + })) + iam := newSharedIdentityIAM() + iam.SetIAMIntegration(NewS3IAMIntegration(mgr, "")) + s3a := &S3ApiServer{iam: iam} + s3a.SetIcebergCredentialRole("arn:aws:iam::role/IcebergTableAccess", 3600) + return s3a +} + +// vend mints table credentials for principal as if its request had been +// signed with accessKey ("" for a request that recorded none); a non-empty +// secret marks the credential as a stored identity's, as a verifier would. +func vend(t *testing.T, s3a *S3ApiServer, principal, accessKey, secret string) *VendedCredentials { + t.Helper() + ctx := context.Background() + if accessKey != "" { + ctx = s3_constants.SetAccessKeyIdInContext(ctx, accessKey) + } + if secret != "" { + ctx = s3_constants.SetStoredAccessKeyIdInContext(ctx, accessKey, secret) + } + creds, err := s3a.VendTableCredentials(ctx, principal, "warehouse", "ns/t", false) + require.NoError(t, err) + require.NotNil(t, creds) + return creds +} + +// use validates a vended session the way a data-path request carrying it is +// validated. +func use(t *testing.T, s3a *S3ApiServer, creds *VendedCredentials) s3err.ErrorCode { + t.Helper() + r, err := newTestRequest("GET", "https://example.com/warehouse/ns/t/data/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := s3a.iam.validateSTSSessionToken(r, creds.SessionToken, creds.AccessKeyID) + return errCode +} + +func sessionInfo(t *testing.T, s3a *S3ApiServer, creds *VendedCredentials) *sts.SessionInfo { + t.Helper() + info, err := s3a.stsService().ValidateSessionToken(context.Background(), creds.SessionToken) + require.NoError(t, err) + return info +} + +func TestVendedSessionEndsWithItsAccessKey(t *testing.T) { + s3a := vendingServer(t) + sessionA := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + sessionB := vend(t, s3a, "shared-user", sharedKeyB, sharedSecretB) + require.Equal(t, s3err.ErrNone, use(t, s3a, sessionA), "control: the session works while its key does") + require.Equal(t, s3err.ErrNone, use(t, s3a, sessionB)) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials[0].Status = iamAccessKeyStatusInactive + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A deactivated") + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionB), "the session vended through key B keeps working") + + identity.Credentials[0].Status = iamAccessKeyStatusActive + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionA), "key A reactivated") + + identity.Credentials[0].Expiration = time.Now().Add(-time.Minute).Unix() + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A expired") + identity.Credentials[0].Expiration = 0 + + identity.Credentials = identity.Credentials[1:] + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, sessionA), "key A deleted") + assert.Equal(t, s3err.ErrNone, use(t, s3a, sessionB), "the session vended through key B keeps working") +} + +func TestVendedSessionEndsWithItsIdentity(t *testing.T) { + s3a := vendingServer(t) + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + require.Equal(t, s3err.ErrNone, use(t, s3a, session)) + + s3a.iam.nameToIdentity["shared-user"].Disabled = true + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "identity disabled") + s3a.iam.nameToIdentity["shared-user"].Disabled = false + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), "identity enabled again") + + delete(s3a.iam.nameToIdentity, "shared-user") + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "identity deleted") +} + +// A caller that is not a stored identity (a caller signing with another STS +// session's key, an OIDC user, a request that recorded no key) still gets +// credentials, unbound as before: binding them would refuse every request +// made with them. +func TestVendedSessionIsBoundOnlyForAStoredIdentitysKey(t *testing.T) { + s3a := vendingServer(t) + control := sessionInfo(t, s3a, vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA)) + assert.Equal(t, sharedKeyA, control.SourceAccessKeyId, "control") + assert.Equal(t, sharedKeyA, control.BoundAccessKeyId, "control") + + for _, tc := range []struct{ principal, key, why string }{ + {"47ad4828c45b3f337bc3146081ba8f0f", "ASIATEMPORARYKEY0001", "an STS session's subject and temporary key"}, + {"shared-user", "ASIATEMPORARYKEY0001", "an STS session whose subject names a stored identity"}, + {"alice@example.com", "", "an OIDC user"}, + {"shared-user", "", "no recorded key"}, + } { + session := vend(t, s3a, tc.principal, tc.key, "") + info := sessionInfo(t, s3a, session) + assert.Equal(t, tc.key, info.SourceAccessKeyId, tc.why) + assert.Empty(t, info.BoundAccessKeyId, tc.why) + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), tc.why) + } +} + +// Tokens minted before the binding existed record the caller's key under the +// audit claim only. Validating them as bound would end live table credentials +// at upgrade — a chained STS caller's key can never resolve to a stored +// credential. They stay valid until expiry, as they always did. +func TestLegacyVendedSessionStaysUnbound(t *testing.T) { + iam := newSharedIdentityIAM() + iam.iamIntegration = &MockIAMIntegration{ + validateSessionFunc: func(ctx context.Context, token string) (*sts.SessionInfo, error) { + return &sts.SessionInfo{ + SessionId: "sid", + AssumedRoleUser: "arn:aws:sts::assumed-role/Reader/session", + Principal: "arn:aws:sts::assumed-role/Reader/session", + IdentityProvider: icebergVendedCredentialsProvider, + ExternalUserId: "someone-not-in-the-identity-table", + SourceAccessKeyId: "ASIATEMPORARYKEY0001", + Credentials: &sts.Credentials{AccessKeyId: "ASIALEGACY", SecretAccessKey: "secret"}, + ExpiresAt: time.Now().Add(time.Hour), + }, nil + }, + } + r, err := newTestRequest("GET", "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := iam.validateSTSSessionToken(r, "token", "ASIALEGACY") + assert.Equal(t, s3err.ErrNone, errCode) +} + +// A key re-created under the same id with a different secret does not revive +// sessions bound to the deleted original: the fingerprint recorded at mint +// time no longer matches. +func TestRecreatedKeyDoesNotReviveVendedSession(t *testing.T) { + s3a := vendingServer(t) + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + require.Equal(t, s3err.ErrNone, use(t, s3a, session)) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials[0] = &Credential{AccessKey: sharedKeyA, SecretKey: "recreated-secret"} + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "same id, different secret") + + identity.Credentials[0] = &Credential{AccessKey: sharedKeyA, SecretKey: sharedSecretA} + assert.Equal(t, s3err.ErrNone, use(t, s3a, session), "the identical credential is indistinguishable") +} + +// A session for a stored identity stays bound even when the key no longer +// resolves at vend time — revoked between the request's authentication and +// the vend, or held by a different identity now. The session dies on its +// first request instead of living unbound until it expires. +func TestVendedSessionStaysBoundWhenTheKeyIsGoneAtVend(t *testing.T) { + s3a := vendingServer(t) + + identity := s3a.iam.nameToIdentity["shared-user"] + identity.Credentials = identity.Credentials[1:] // sharedKeyA revoked mid-request + session := vend(t, s3a, "shared-user", sharedKeyA, sharedSecretA) + assert.Equal(t, sharedKeyA, sessionInfo(t, s3a, session).SourceAccessKeyId, "still bound") + assert.Equal(t, s3err.ErrInvalidAccessKeyID, use(t, s3a, session), "the dead key's session is refused") +} + +// Only sessions the catalog vended are bound; any other STS session (web +// identity, AssumeRole) is validated as before. +func TestOtherSessionsAreNotBoundToAKey(t *testing.T) { + iam := newSharedIdentityIAM() + iam.iamIntegration = &MockIAMIntegration{ + validateSessionFunc: func(ctx context.Context, token string) (*sts.SessionInfo, error) { + return &sts.SessionInfo{ + SessionId: "sid", + AssumedRoleUser: "arn:aws:sts::assumed-role/Reader/session", + Principal: "arn:aws:sts::assumed-role/Reader/session", + IdentityProvider: "oidc-provider", + ExternalUserId: "someone-not-in-the-identity-table", + SourceAccessKeyId: "AKIAGONE", + Credentials: &sts.Credentials{AccessKeyId: "ASIAWEBID", SecretAccessKey: "secret"}, + ExpiresAt: time.Now().Add(time.Hour), + }, nil + }, + } + r, err := newTestRequest("GET", "https://example.com/bucket/f.parquet", 0, nil) + require.NoError(t, err) + _, _, errCode := iam.validateSTSSessionToken(r, "token", "ASIAWEBID") + assert.Equal(t, s3err.ErrNone, errCode) +} + +// The check on the request path is one read-locked map lookup and a scan of +// one identity's keys. +func BenchmarkVendedSessionKeyLive(b *testing.B) { + iam := newSharedIdentityIAM() + for i := 0; i < 1000; i++ { + name := fmt.Sprintf("user-%04d", i) + iam.nameToIdentity[name] = &Identity{Name: name, Credentials: []*Credential{{AccessKey: name}}} + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if iam.vendedSessionKeyLive("shared-user", sharedKeyB, "") != s3err.ErrNone { + b.Fatal("refused") + } + } +} From 7c478fd89ad96fa5c0389c3d41f623e4a38904d3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 04:33:14 -0700 Subject: [PATCH 05/70] fuse integration: run posix suites on weed mini and allow allow_other mounts (#11984) The separate master/volume/filer processes in framework.go bound fixed ports, discarded child output, and never noticed an early process exit, so a slow or dead volume server burned the whole 30s timeout with no logs. Switch to the same weed-mini approach the fuse_test package framework already uses: dynamic filer ports, child output captured to log files, and readiness waits that bail out the moment the process dies. The external-mount path (TEST_MOUNT_POINT) is unchanged. The posix-compliance workflow mounts with -allowOthers, but fusermount refuses allow_other unless user_allow_other is set in /etc/fuse.conf, mirroring the fuse-integration workflow FUSE setup. --- .github/workflows/posix-compliance.yml | 10 +- test/fuse_integration/framework.go | 349 ++++++++++++++++--------- 2 files changed, 231 insertions(+), 128 deletions(-) diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml index fc93694466..973fc564bb 100644 --- a/.github/workflows/posix-compliance.yml +++ b/.github/workflows/posix-compliance.yml @@ -98,7 +98,11 @@ jobs: # Set permissions on FUSE device sudo chmod 666 /dev/fuse - + + # Allow non-root mounts to pass -o allow_other (weed -allowOthers) + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf + sudo chmod 644 /etc/fuse.conf + # Ensure fuse module is loaded sudo modprobe fuse || true @@ -680,6 +684,8 @@ jobs: sudo apt-get install -y fuse libfuse-dev fio sudo usermod -a -G fuse $USER sudo chmod 666 /dev/fuse + # The posix suites mount with -allowOthers; allow_other needs this + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf - name: Build SeaweedFS run: make @@ -729,6 +735,8 @@ jobs: sudo apt-get update sudo apt-get install -y fuse libfuse-dev pip3 install --user nfstest + # The posix suites mount with -allowOthers; allow_other needs this + echo 'user_allow_other' | sudo tee -a /etc/fuse.conf - name: Build SeaweedFS run: make diff --git a/test/fuse_integration/framework.go b/test/fuse_integration/framework.go index 8424fb1d79..47280e652c 100644 --- a/test/fuse_integration/framework.go +++ b/test/fuse_integration/framework.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "syscall" "testing" "time" @@ -14,19 +15,19 @@ import ( "github.com/stretchr/testify/require" ) -// FuseTestFramework provides utilities for FUSE integration testing +// FuseTestFramework provides utilities for FUSE integration testing. +// It starts a single "weed mini" process (master+volume+filer in one) +// and a separate "weed mount" process for the FUSE filesystem. type FuseTestFramework struct { t *testing.T tempDir string mountPoint string dataDir string - masterProcess *os.Process - volumeProcess *os.Process - filerProcess *os.Process - mountProcess *os.Process - masterAddr string - volumeAddr string + logDir string + miniProcess *managedProcess + mountProcess *managedProcess filerAddr string + filerPort int weedBinary string isSetup bool config *TestConfig @@ -35,14 +36,18 @@ type FuseTestFramework struct { // TestConfig holds configuration for FUSE tests type TestConfig struct { - Collection string - Replication string - ChunkSizeMB int - CacheSizeMB int - NumVolumes int - EnableDebug bool - MountOptions []string - SkipCleanup bool // for debugging failed tests + Collection string + Replication string + ChunkSizeMB int + CacheSizeMB int + NumVolumes int + EnableDebug bool + // MountGlobalOptions are glog flags (-v, -logtostderr, ...) registered on + // weed's global flagset; they must precede the subcommand name or the + // mount process dies at flag parsing. + MountGlobalOptions []string + MountOptions []string + SkipCleanup bool // for debugging failed tests } // DefaultTestConfig returns a default configuration for FUSE tests @@ -59,7 +64,7 @@ func DefaultTestConfig() *TestConfig { } } -// NewFuseTestFramework creates a new FUSE testing framework +// NewFuseTestFramework creates a new FUSE testing framework. func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { if config == nil { config = DefaultTestConfig() @@ -68,20 +73,63 @@ func NewFuseTestFramework(t *testing.T, config *TestConfig) *FuseTestFramework { tempDir, err := os.MkdirTemp("", "seaweedfs_fuse_test_") require.NoError(t, err) + filerPort := freePort(t) + return &FuseTestFramework{ t: t, tempDir: tempDir, mountPoint: filepath.Join(tempDir, "mount"), dataDir: filepath.Join(tempDir, "data"), - masterAddr: "127.0.0.1:19333", - volumeAddr: "127.0.0.1:18080", - filerAddr: "127.0.0.1:18888", + logDir: filepath.Join(tempDir, "logs"), + filerPort: filerPort, + filerAddr: fmt.Sprintf("127.0.0.1:%d", filerPort), weedBinary: findWeedBinary(), isSetup: false, config: config, } } +// grpcPortOffset mirrors weed's HTTP->gRPC port convention (gRPC = HTTP + offset); +// "weed mount" derives the filer gRPC port from the filer HTTP address the same way. +const grpcPortOffset = 10000 + +// freePort returns a free filer HTTP port whose gRPC sibling (port+grpcPortOffset) +// is also free. Both must stay below the Linux ephemeral floor (32768): a gRPC port +// above it can be transiently grabbed by an outbound connection, forcing mini to +// relocate its filer gRPC port while "weed mount" keeps dialing HTTP+10000 — the +// mount then never connects and the test times out. Capping HTTP at 22000 keeps the +// gRPC port at or below 32000. +func freePort(t *testing.T) int { + t.Helper() + const ( + minServicePort = 20000 + maxServicePort = 22000 + ) + + portCount := maxServicePort - minServicePort + 1 + start := minServicePort + int(time.Now().UnixNano()%int64(portCount)) + + for attempt := 0; attempt < portCount; attempt++ { + port := minServicePort + (start-minServicePort+attempt)%portCount + if portIsFree(port) && portIsFree(port+grpcPortOffset) { + return port + } + } + + t.Fatalf("failed to allocate a free HTTP/gRPC port pair in [%d,%d]", minServicePort, maxServicePort) + return 0 +} + +// portIsFree reports whether a TCP port can currently be bound on 127.0.0.1. +func portIsFree(port int) bool { + l, err := net.Listen("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + return false + } + l.Close() + return true +} + // Setup starts SeaweedFS cluster and mounts FUSE filesystem func (f *FuseTestFramework) Setup(config *TestConfig) error { if f.isSetup { @@ -159,41 +207,22 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Create directories for full cluster setup - dirs := []string{f.mountPoint, f.dataDir} + dirs := []string{f.mountPoint, f.logDir, f.dataDir} for _, dir := range dirs { if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("failed to create directory %s: %v", dir, err) } } - // Start master - if err := f.startMaster(config); err != nil { - return fmt.Errorf("failed to start master: %v", err) + // Start weed mini (master + volume + filer in one process) + if err := f.startMini(config); err != nil { + return fmt.Errorf("failed to start weed mini: %v", err) } - // Wait for master to be ready - if err := f.waitForService(f.masterAddr, 30*time.Second); err != nil { - return fmt.Errorf("master not ready: %v", err) - } - - // Start volume servers - if err := f.startVolumeServers(config); err != nil { - return fmt.Errorf("failed to start volume servers: %v", err) - } - - // Wait for volume server to be ready - if err := f.waitForService(f.volumeAddr, 30*time.Second); err != nil { - return fmt.Errorf("volume server not ready: %v", err) - } - - // Start filer - if err := f.startFiler(config); err != nil { - return fmt.Errorf("failed to start filer: %v", err) - } - - // Wait for filer to be ready - if err := f.waitForService(f.filerAddr, 30*time.Second); err != nil { - return fmt.Errorf("filer not ready: %v", err) + // Wait for filer to be ready (mini starts all services on filerPort) + if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil { + f.dumpLog("mini") + return fmt.Errorf("weed mini not ready: %v", err) } // Skip rather than fail on a host with no FUSE device to mount through @@ -208,6 +237,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { // Wait for mount to be ready if err := f.waitForMount(30 * time.Second); err != nil { + f.dumpLog("mount") return fmt.Errorf("FUSE mount not ready: %v", err) } @@ -215,7 +245,8 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { return nil } -// Cleanup stops all processes and removes temporary files +// Cleanup stops all processes and removes temporary files. +// If the test failed, it dumps logs automatically. func (f *FuseTestFramework) Cleanup() { // An external mount needs no teardown; only this run's work dir is ours. // A TEST_SKIP_CLUSTER_SETUP that fell through to a self-started cluster @@ -228,25 +259,32 @@ func (f *FuseTestFramework) Cleanup() { return } - if f.mountProcess != nil { - f.unmountFuse() + if f.t.Failed() { + f.DumpLogs() } // Stop processes in reverse order - processes := []*os.Process{f.mountProcess, f.filerProcess, f.volumeProcess, f.masterProcess} - for _, proc := range processes { - if proc != nil { - proc.Signal(syscall.SIGTERM) - proc.Wait() - } + f.unmountFuse() + if f.miniProcess != nil { + f.miniProcess.stop() + f.miniProcess = nil } + f.copyLogsForCI() + // Remove temp directory if !f.config.SkipCleanup { os.RemoveAll(f.tempDir) } } +// DumpLogs prints the tail of all SeaweedFS process logs to test output. +func (f *FuseTestFramework) DumpLogs() { + for _, name := range []string{"mini", "mount"} { + f.dumpLog(name) + } +} + // GetMountPoint returns the FUSE mount point path func (f *FuseTestFramework) GetMountPoint() string { return f.mountPoint @@ -257,88 +295,136 @@ func (f *FuseTestFramework) GetFilerAddr() string { return f.filerAddr } -// startMaster starts the SeaweedFS master server -func (f *FuseTestFramework) startMaster(config *TestConfig) error { +// managedProcess is a started weed sub-command whose exit is watched, so a wait +// for it to come up ends the moment it dies instead of burning its full timeout. +type managedProcess struct { + cmd *exec.Cmd + done chan struct{} + err error // exit error, read only after done is closed +} + +// exited returns the exit error once the process is gone, nil while it runs. +func (p *managedProcess) exited() error { + select { + case <-p.done: + if p.err != nil { + return p.err + } + return fmt.Errorf("exit status 0") + default: + return nil + } +} + +// stop asks the process to terminate and waits for it to go away. +func (p *managedProcess) stop() { + // Signal fails with os.ErrProcessDone when the child is already gone, which + // is exactly the case the select below handles. + _ = p.cmd.Process.Signal(syscall.SIGTERM) + select { + case <-p.done: + case <-time.After(10 * time.Second): + p.cmd.Process.Kill() + <-p.done + } +} + +// startProcess is a helper that starts a weed sub-command with output captured +// to a log file in f.logDir. +func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) { + logFile, err := os.Create(filepath.Join(f.logDir, name+".log")) + if err != nil { + return nil, fmt.Errorf("create log file: %v", err) + } + cmd := exec.Command(f.weedBinary, args...) + cmd.Dir = f.tempDir + cmd.Stdout = logFile + cmd.Stderr = logFile + if err := cmd.Start(); err != nil { + logFile.Close() + return nil, err + } + // Close the file handle — the child process inherited it. + logFile.Close() + + p := &managedProcess{cmd: cmd, done: make(chan struct{})} + go func() { + p.err = cmd.Wait() + close(p.done) + }() + return p, nil +} + +// dumpLog prints the last lines of a process log file to the test output +// for debugging when a service fails to start or a test fails. +func (f *FuseTestFramework) dumpLog(name string) { + data, err := os.ReadFile(filepath.Join(f.logDir, name+".log")) + if err != nil { + f.t.Logf("[%s log] (not available: %v)", name, err) + return + } + // Show last 16KB on failure for meaningful context. + const maxTail = 16 * 1024 + if len(data) > maxTail { + data = data[len(data)-maxTail:] + } + f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data)) +} + +// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/ +// so the CI workflow can upload them as artifacts. +func (f *FuseTestFramework) copyLogsForCI() { + ciLogDir := "/tmp/seaweedfs-fuse-logs" + os.MkdirAll(ciLogDir, 0755) + for _, name := range []string{"mini", "mount"} { + src := filepath.Join(f.logDir, name+".log") + data, err := os.ReadFile(src) + if err != nil { + continue + } + os.WriteFile(filepath.Join(ciLogDir, name+".log"), data, 0644) + } +} + +// startMini starts "weed mini" which runs master+volume+filer in one process. +func (f *FuseTestFramework) startMini(config *TestConfig) error { var args []string if config.EnableDebug { // -v is a global flag; before the subcommand or flag parsing dies args = append(args, "-v=4") } args = append(args, - "master", + "mini", + "-dir="+f.dataDir, "-ip=127.0.0.1", - "-port=19333", - "-mdir=" + filepath.Join(f.dataDir, "master"), - "-raftBootstrap", + "-ip.bind=127.0.0.1", + "-filer.port="+strconv.Itoa(f.filerPort), + "-s3=false", + "-webdav=false", + "-admin.ui=false", ) - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { + proc, err := f.startProcess("mini", args) + if err != nil { return err } - f.masterProcess = cmd.Process - return nil -} - -// startVolumeServers starts SeaweedFS volume servers -func (f *FuseTestFramework) startVolumeServers(config *TestConfig) error { - var args []string - if config.EnableDebug { - args = append(args, "-v=4") - } - args = append(args, - "volume", - "-mserver=" + f.masterAddr, - "-ip=127.0.0.1", - "-port=18080", - "-dir=" + filepath.Join(f.dataDir, "volume"), - fmt.Sprintf("-max=%d", config.NumVolumes), - ) - - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { - return err - } - f.volumeProcess = cmd.Process - return nil -} - -// startFiler starts the SeaweedFS filer server -func (f *FuseTestFramework) startFiler(config *TestConfig) error { - var args []string - if config.EnableDebug { - args = append(args, "-v=4") - } - args = append(args, - "filer", - "-master=" + f.masterAddr, - "-ip=127.0.0.1", - "-port=18888", - ) - - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { - return err - } - f.filerProcess = cmd.Process + f.miniProcess = proc return nil } // mountFuse mounts the SeaweedFS FUSE filesystem func (f *FuseTestFramework) mountFuse(config *TestConfig) error { - var args []string + args := append([]string{}, config.MountGlobalOptions...) if config.EnableDebug { args = append(args, "-v=4") } args = append(args, "mount", - "-filer=" + f.filerAddr, - "-dir=" + f.mountPoint, + "-filer="+f.filerAddr, + "-dir="+f.mountPoint, "-filer.path=/", "-dirAutoCreate", + "-allowOthers=false", ) if config.Collection != "" { @@ -351,35 +437,35 @@ func (f *FuseTestFramework) mountFuse(config *TestConfig) error { args = append(args, fmt.Sprintf("-chunkSizeLimitMB=%d", config.ChunkSizeMB)) } if config.CacheSizeMB > 0 { - args = append(args, fmt.Sprintf("-cacheSizeMB=%d", config.CacheSizeMB)) + args = append(args, fmt.Sprintf("-cacheCapacityMB=%d", config.CacheSizeMB)) } args = append(args, config.MountOptions...) - cmd := exec.Command(f.weedBinary, args...) - cmd.Dir = f.tempDir - if err := cmd.Start(); err != nil { + proc, err := f.startProcess("mount", args) + if err != nil { return err } - f.mountProcess = cmd.Process + f.mountProcess = proc return nil } // unmountFuse unmounts the FUSE filesystem func (f *FuseTestFramework) unmountFuse() error { if f.mountProcess != nil { - f.mountProcess.Signal(syscall.SIGTERM) - f.mountProcess.Wait() + f.mountProcess.stop() f.mountProcess = nil } // Also try system unmount as backup + exec.Command("fusermount3", "-u", f.mountPoint).Run() + exec.Command("fusermount", "-u", f.mountPoint).Run() exec.Command("umount", f.mountPoint).Run() return nil } // waitForService waits for a service to be available -func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) error { +func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error { deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { conn, err := net.DialTimeout("tcp", addr, 1*time.Second) @@ -387,19 +473,23 @@ func (f *FuseTestFramework) waitForService(addr string, timeout time.Duration) e conn.Close() return nil } + if exitErr := proc.exited(); exitErr != nil { + return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr) + } time.Sleep(100 * time.Millisecond) } return fmt.Errorf("service at %s not ready within timeout", addr) } -// waitForMount waits for the FUSE mount to be ready +// waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe +// alone is not enough: the bare mount point directory passes both before the +// mount process finishes starting, letting tests race ahead and write to the +// local disk underneath the mount. The mount point's device ID differing from +// its parent's confirms a filesystem is actually mounted there. func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { - // The mount point is created before weed mount starts, so Stat/ReadDir - // alone succeed on the plain directory. The device id only differs from - // its parent's once a filesystem is actually mounted on it. parentDev, err := deviceID(filepath.Dir(f.mountPoint)) if err != nil { - return fmt.Errorf("stat mount parent: %w", err) + return fmt.Errorf("stat mount point parent: %w", err) } deadline := time.Now().Add(timeout) for time.Now().Before(deadline) { @@ -408,6 +498,11 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { return nil } } + // A mount that cannot mount at all (no /dev/fuse, fusermount not setuid) + // dies within a second; reporting that beats waiting out the timeout. + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v)", exitErr) + } time.Sleep(100 * time.Millisecond) } return fmt.Errorf("mount point not ready within timeout") From f7af816ab415dcf6da6a834dbd4269db486388ef Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 04:49:52 -0700 Subject: [PATCH 06/70] ci: cancel superseded workflow runs on PR pushes (#11979) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: cancel superseded workflow runs on PR pushes PR-triggered workflows without a concurrency group keep running (and holding runner slots) after a force-push or quick follow-up commit supersedes them. Add the same group/cancel-in-progress block the go workflow already uses, keyed on head_ref, to the remaining PR workflows. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: key concurrency groups on PR number or ref, not head_ref github.head_ref is empty on tag pushes and manual runs (every run then shares one group and cancels the last) and collides across forks when two PRs share a branch name. pull_request.number is unique per PR; github.ref covers push/tag/dispatch. Applied to all workflows using the pattern, including pre-existing ones. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: restrict cancel-in-progress to PR and branch-push runs * ci: serialize container_latest manual runs by destination tag Two workflow_dispatch runs sharing an image_tag could otherwise build concurrently: their per-arch images publish to the same platform tags, so one run's create-manifest job can combine platforms from different source refs. Group dispatch runs by the destination tag (with cancel-in-progress already disabled for non-PR/non-branch events) so same-tag publishes queue instead of interleaving. * ci: serialize manual publish runs by destination tag in remaining container workflows * ci: group foundationdb release dispatch runs by release tag A manual run now shares the tag-push concurrency group for the same release, so a re-publish of a tag queues behind (or ahead of) the push run instead of racing it on the same image tag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: serialize foundationdb_version dispatch runs by destination tag Two manual runs resolving to the same docker tag now share a concurrency group (explicit image_tag, else the computed foundationdb__seaweedfs_ default), so they queue instead of racing pushes to the same tag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: drop unsupported toLower() from foundationdb_version concurrency group * ci: key rocksdb_version dispatch group on its published tag * ci: queue publish runs via turnstyle instead of cancel-free shared groups * Revert "ci: queue publish runs via turnstyle instead of cancel-free shared groups" This reverts commit 039148f8d59a7711bf9bddc7dcb2b2e95d328189. * ci: queue same-tag publish runs instead of replacing the pending one Dispatch/tag-only workflows can never cancel in progress, so opt their concurrency groups into queue: max — up to 100 same-tag publishes wait in line rather than each new dispatch cancelling the pending one. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 4 ++-- .github/workflows/codespell.yml | 4 ++++ .github/workflows/container_dev.yml | 4 ++++ .github/workflows/container_foundationdb_version.yml | 4 ++++ .github/workflows/container_latest.yml | 5 +++++ .github/workflows/container_release_foundationdb.yml | 5 +++++ .github/workflows/container_rocksdb_version.yml | 5 +++++ .github/workflows/depsreview.yml | 4 ++++ .github/workflows/e2e.yml | 4 ++-- .github/workflows/ec-integration-tests.yml | 4 ++++ .github/workflows/fuse-dlm-integration.yml | 4 ++-- .github/workflows/fuse-failover.yml | 4 ++-- .github/workflows/fuse-integration.yml | 4 ++-- .github/workflows/fuse-p2p-integration.yml | 4 ++-- .github/workflows/go.yml | 4 ++-- .github/workflows/helm_ci.yml | 4 ++++ .github/workflows/java_integration_tests.yml | 4 ++++ .github/workflows/java_unit_tests.yml | 4 ++++ .github/workflows/kafka-quicktest.yml | 4 ++-- .github/workflows/kafka-tests.yml | 4 ++-- .github/workflows/kms-tests.yml | 4 ++-- .github/workflows/master-cold-start-tests.yml | 4 ++-- .github/workflows/metadata-subscribe-tests.yml | 4 ++-- .github/workflows/mount-benchmark.yml | 4 ++-- .github/workflows/mount-windows-conformance.yml | 4 ++-- .github/workflows/mount-windows.yml | 4 ++-- .github/workflows/multi-master-tests.yml | 4 ++-- .github/workflows/performance.yml | 4 ++-- .github/workflows/plugin-workers.yml | 4 ++++ .github/workflows/postgres-tests.yml | 4 ++-- .github/workflows/rust-volume-server-tests.yml | 4 ++-- .github/workflows/rust-worker-tests.yml | 4 ++-- .github/workflows/s3-etag-acl-tests.yml | 4 ++-- .github/workflows/s3-example-integration-tests.yml | 4 ++-- .github/workflows/s3-filer-group-tests.yml | 4 ++-- .github/workflows/s3-go-tests.yml | 4 ++-- .github/workflows/s3-iam-tests.yml | 4 ++-- .github/workflows/s3-keycloak-tests.yml | 4 ++-- .github/workflows/s3-mutation-regression-tests.yml | 4 ++-- .github/workflows/s3-parquet-tests.yml | 4 ++++ .github/workflows/s3-policy-tests.yml | 4 ++-- .github/workflows/s3-proxy-signature-tests.yml | 4 ++-- .github/workflows/s3-sdk-v2-routing-tests.yml | 4 ++-- .github/workflows/s3-spark-tests.yml | 4 ++-- .github/workflows/s3-sse-tests.yml | 4 ++-- .github/workflows/s3-tables-tests.yml | 4 ++++ .github/workflows/s3tests.yml | 4 ++-- .github/workflows/sftp-tests.yml | 4 ++-- .github/workflows/spark-integration-tests.yml | 4 ++++ .github/workflows/star_history.yml | 9 ++++----- .github/workflows/telemetry-integration.yml | 4 ++++ .github/workflows/terraform_ci.yml | 4 ++++ .github/workflows/test-s3-over-https-using-awscli.yml | 4 ++++ .github/workflows/tls-rotation-tests.yml | 4 ++++ .github/workflows/tus-tests.yml | 4 ++-- .github/workflows/vacuum-integration-tests.yml | 4 ++++ .github/workflows/volume-server-integration-tests.yml | 4 ++-- 57 files changed, 159 insertions(+), 77 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index d8545097d0..edc2af3c98 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,8 +9,8 @@ on: - '.github/workflows/codeql.yml' concurrency: - group: ${{ github.head_ref }}/codeql - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} jobs: CodeQL-Build: diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 9c4d99a79e..a7be2ead3b 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -8,6 +8,10 @@ on: pull_request: branches: [master] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index f20820a59f..f5d880a8ec 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -13,6 +13,10 @@ on: - 'go.sum' - '.github/workflows/container_dev.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/container_foundationdb_version.yml b/.github/workflows/container_foundationdb_version.yml index 6fa0149b76..6903dcf2c0 100644 --- a/.github/workflows/container_foundationdb_version.yml +++ b/.github/workflows/container_foundationdb_version.yml @@ -24,6 +24,10 @@ on: required: false default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || format('foundationdb_{0}_seaweedfs_{1}', github.event.inputs.fdb_version || '7.4.5', github.event.inputs.seaweedfs_ref || github.ref_name) }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 7a47628824..3d99baaf24 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -30,6 +30,11 @@ on: type: boolean default: false +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.event.inputs.image_tag || github.run_id }} + cancel-in-progress: false + queue: max + permissions: contents: read security-events: write diff --git a/.github/workflows/container_release_foundationdb.yml b/.github/workflows/container_release_foundationdb.yml index bf1270b0a8..2050d99604 100644 --- a/.github/workflows/container_release_foundationdb.yml +++ b/.github/workflows/container_release_foundationdb.yml @@ -11,6 +11,11 @@ on: required: true default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || format('refs/tags/{0}', github.event.inputs.release_tag) || github.run_id }} + cancel-in-progress: false + queue: max + permissions: contents: read diff --git a/.github/workflows/container_rocksdb_version.yml b/.github/workflows/container_rocksdb_version.yml index 89b4e82049..f06eb500f4 100644 --- a/.github/workflows/container_rocksdb_version.yml +++ b/.github/workflows/container_rocksdb_version.yml @@ -16,6 +16,11 @@ on: required: false default: '' +concurrency: + group: ${{ github.workflow }}-${{ github.event.inputs.image_tag || format('rocksdb_{0}_seaweedfs_{1}', github.event.inputs.rocksdb_version, github.event.inputs.seaweedfs_ref) }} + cancel-in-progress: false + queue: max + permissions: contents: read diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index e4338c240a..0ba5dbcd2c 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -1,6 +1,10 @@ name: 'Dependency Review' on: [pull_request] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0e58880119..ef5daf28bd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -19,8 +19,8 @@ on: - '.github/workflows/e2e.yml' concurrency: - group: ${{ github.head_ref }}/e2e - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/ec-integration-tests.yml b/.github/workflows/ec-integration-tests.yml index e078e01e6c..ec26d446eb 100644 --- a/.github/workflows/ec-integration-tests.yml +++ b/.github/workflows/ec-integration-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/ec-integration-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/fuse-dlm-integration.yml b/.github/workflows/fuse-dlm-integration.yml index 47a27cc199..f4a7931d8b 100644 --- a/.github/workflows/fuse-dlm-integration.yml +++ b/.github/workflows/fuse-dlm-integration.yml @@ -19,8 +19,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-dlm-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml index c11ca3e2f1..e73d588ba4 100644 --- a/.github/workflows/fuse-failover.yml +++ b/.github/workflows/fuse-failover.yml @@ -24,8 +24,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-failover - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/fuse-integration.yml b/.github/workflows/fuse-integration.yml index 3b23cb2228..8482eb287d 100644 --- a/.github/workflows/fuse-integration.yml +++ b/.github/workflows/fuse-integration.yml @@ -17,8 +17,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref }}/fuse-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 28771e11a9..8cb5f2eca7 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -25,8 +25,8 @@ on: - '.github/actions/fix-fusermount-setuid/**' concurrency: - group: ${{ github.head_ref || github.ref }}/fuse-p2p-integration - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index b2e5b52979..ffffaee697 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/go.yml' concurrency: - group: ${{ github.head_ref }}/go - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/helm_ci.yml b/.github/workflows/helm_ci.yml index ce0ecf916b..0a75617d79 100644 --- a/.github/workflows/helm_ci.yml +++ b/.github/workflows/helm_ci.yml @@ -8,6 +8,10 @@ on: branches: [ master ] paths: ['k8s/**', '.github/workflows/helm_ci.yml', '.github/scripts/helm_jwt_keys.py'] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml index 075df5f857..9dc3461c82 100644 --- a/.github/workflows/java_integration_tests.yml +++ b/.github/workflows/java_integration_tests.yml @@ -14,6 +14,10 @@ on: - 'weed/**' - '.github/workflows/java_integration_tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: test: name: Java Integration Tests diff --git a/.github/workflows/java_unit_tests.yml b/.github/workflows/java_unit_tests.yml index 753d1708e8..c34f49b687 100644 --- a/.github/workflows/java_unit_tests.yml +++ b/.github/workflows/java_unit_tests.yml @@ -12,6 +12,10 @@ on: - 'other/java/**' - '.github/workflows/java_unit_tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: test: name: Java Unit Tests diff --git a/.github/workflows/kafka-quicktest.yml b/.github/workflows/kafka-quicktest.yml index c18a87048b..165028635a 100644 --- a/.github/workflows/kafka-quicktest.yml +++ b/.github/workflows/kafka-quicktest.yml @@ -24,8 +24,8 @@ on: workflow_dispatch: # Allow manual trigger concurrency: - group: ${{ github.head_ref }}/kafka-quicktest - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index 7a40ee3583..c0e15cee89 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -23,8 +23,8 @@ on: - '.github/workflows/kafka-tests.yml' concurrency: - group: ${{ github.head_ref }}/kafka-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} # Force different runners for better isolation env: diff --git a/.github/workflows/kms-tests.yml b/.github/workflows/kms-tests.yml index 3c997fd523..6e020f0dea 100644 --- a/.github/workflows/kms-tests.yml +++ b/.github/workflows/kms-tests.yml @@ -19,8 +19,8 @@ on: - 'test/kms/**' concurrency: - group: ${{ github.head_ref || github.ref }}-kms-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/master-cold-start-tests.yml b/.github/workflows/master-cold-start-tests.yml index bd0ed72243..bd983c7c84 100644 --- a/.github/workflows/master-cold-start-tests.yml +++ b/.github/workflows/master-cold-start-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/master-cold-start-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/master-cold-start-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/metadata-subscribe-tests.yml b/.github/workflows/metadata-subscribe-tests.yml index b47c16b9cd..e674986b0d 100644 --- a/.github/workflows/metadata-subscribe-tests.yml +++ b/.github/workflows/metadata-subscribe-tests.yml @@ -25,8 +25,8 @@ on: - '.github/workflows/metadata-subscribe-tests.yml' concurrency: - group: ${{ github.head_ref }}/metadata-subscribe-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/mount-benchmark.yml b/.github/workflows/mount-benchmark.yml index 90310ee6cf..52c7dcb49d 100644 --- a/.github/workflows/mount-benchmark.yml +++ b/.github/workflows/mount-benchmark.yml @@ -10,8 +10,8 @@ on: branches: [ 'winfsp-bench**' ] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/mount-windows-conformance.yml b/.github/workflows/mount-windows-conformance.yml index 6d490c5d8a..c9aab26a02 100644 --- a/.github/workflows/mount-windows-conformance.yml +++ b/.github/workflows/mount-windows-conformance.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/mount-windows-conformance.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/mount-windows.yml b/.github/workflows/mount-windows.yml index 91f61d8093..18cac2773a 100644 --- a/.github/workflows/mount-windows.yml +++ b/.github/workflows/mount-windows.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/mount-windows.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/multi-master-tests.yml b/.github/workflows/multi-master-tests.yml index 2e0cfe8d95..b42d507f09 100644 --- a/.github/workflows/multi-master-tests.yml +++ b/.github/workflows/multi-master-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/multi-master-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/multi-master-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 95750f6891..8dd438382c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,8 +45,8 @@ on: type: string concurrency: - group: ${{ github.head_ref || github.ref }}/performance - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/plugin-workers.yml b/.github/workflows/plugin-workers.yml index b3f10551c4..94fb1fbf60 100644 --- a/.github/workflows/plugin-workers.yml +++ b/.github/workflows/plugin-workers.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/plugin-workers.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml index a0369b134e..d3eff2fe15 100644 --- a/.github/workflows/postgres-tests.yml +++ b/.github/workflows/postgres-tests.yml @@ -23,8 +23,8 @@ on: - '.github/workflows/postgres-tests.yml' concurrency: - group: ${{ github.head_ref }}/postgres-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index fd68db1a5d..b88a245876 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/rust-volume-server-tests.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml index d2fde3ed0d..66510cf101 100644 --- a/.github/workflows/rust-worker-tests.yml +++ b/.github/workflows/rust-worker-tests.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/rust-worker-tests.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-etag-acl-tests.yml b/.github/workflows/s3-etag-acl-tests.yml index 12425f6f70..13fb6cffb9 100644 --- a/.github/workflows/s3-etag-acl-tests.yml +++ b/.github/workflows/s3-etag-acl-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/s3-etag-acl-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-etag-acl-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-example-integration-tests.yml b/.github/workflows/s3-example-integration-tests.yml index 4e39c3bc07..1765b2f818 100644 --- a/.github/workflows/s3-example-integration-tests.yml +++ b/.github/workflows/s3-example-integration-tests.yml @@ -13,8 +13,8 @@ on: - '.github/workflows/s3-example-integration-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-integration-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml index bda1ac2bca..2fcfc8e692 100644 --- a/.github/workflows/s3-filer-group-tests.yml +++ b/.github/workflows/s3-filer-group-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-filer-group-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-filer-group-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 3a5c07395b..168d75efc3 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-go-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-go-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml index 121f835083..5bf89d7e70 100644 --- a/.github/workflows/s3-iam-tests.yml +++ b/.github/workflows/s3-iam-tests.yml @@ -20,8 +20,8 @@ on: - '.github/workflows/s3-iam-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-iam-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml index d2677431c1..a5a34d45f3 100644 --- a/.github/workflows/s3-keycloak-tests.yml +++ b/.github/workflows/s3-keycloak-tests.yml @@ -16,8 +16,8 @@ on: - '.github/workflows/s3-keycloak-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-keycloak-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-mutation-regression-tests.yml b/.github/workflows/s3-mutation-regression-tests.yml index acf09a8f27..0f30ef4edb 100644 --- a/.github/workflows/s3-mutation-regression-tests.yml +++ b/.github/workflows/s3-mutation-regression-tests.yml @@ -12,8 +12,8 @@ on: - '.github/workflows/s3-mutation-regression-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-mutation-regression-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-parquet-tests.yml b/.github/workflows/s3-parquet-tests.yml index b86c3869e4..20403a533d 100644 --- a/.github/workflows/s3-parquet-tests.yml +++ b/.github/workflows/s3-parquet-tests.yml @@ -23,6 +23,10 @@ env: S3_ENDPOINT_URL: http://localhost:8333 BUCKET_NAME: test-parquet-bucket +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: parquet-integration-tests: name: PyArrow Parquet Tests (Python ${{ matrix.python-version }}) diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml index 7bb900c150..99c1949b18 100644 --- a/.github/workflows/s3-policy-tests.yml +++ b/.github/workflows/s3-policy-tests.yml @@ -24,8 +24,8 @@ on: - '.github/workflows/s3-policy-tests.yml' concurrency: - group: ${{ github.head_ref }}/s3-policy-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-proxy-signature-tests.yml b/.github/workflows/s3-proxy-signature-tests.yml index a8cc6c3679..40756e0982 100644 --- a/.github/workflows/s3-proxy-signature-tests.yml +++ b/.github/workflows/s3-proxy-signature-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/s3-proxy-signature-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-proxy-signature-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-sdk-v2-routing-tests.yml b/.github/workflows/s3-sdk-v2-routing-tests.yml index e63528e145..477c5ef7fd 100644 --- a/.github/workflows/s3-sdk-v2-routing-tests.yml +++ b/.github/workflows/s3-sdk-v2-routing-tests.yml @@ -15,8 +15,8 @@ on: - '.github/workflows/s3-sdk-v2-routing-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/s3-sdk-v2-routing-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index c8682992de..b6e7580a8c 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -11,8 +11,8 @@ on: workflow_dispatch: concurrency: - group: ${{ github.head_ref }}/s3-spark-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index d1e2102203..9dfe3f8758 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -23,8 +23,8 @@ on: - 'test/s3/sse/**' concurrency: - group: ${{ github.head_ref }}/s3-sse-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index 19c1395fe6..275098d960 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -12,6 +12,10 @@ on: - 'go.sum' - '.github/workflows/s3-tables-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 7d481a3f63..353009c4ce 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -25,8 +25,8 @@ on: - '.github/workflows/s3tests.yml' concurrency: - group: ${{ github.head_ref }}/s3tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/sftp-tests.yml b/.github/workflows/sftp-tests.yml index 377608dcf3..9782d4e240 100644 --- a/.github/workflows/sftp-tests.yml +++ b/.github/workflows/sftp-tests.yml @@ -17,8 +17,8 @@ on: - '.github/workflows/sftp-tests.yml' concurrency: - group: ${{ github.head_ref }}/sftp-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml index 9a18898e2b..20c2830e34 100644 --- a/.github/workflows/spark-integration-tests.yml +++ b/.github/workflows/spark-integration-tests.yml @@ -15,6 +15,10 @@ on: - '.github/workflows/spark-integration-tests.yml' workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read checks: write diff --git a/.github/workflows/star_history.yml b/.github/workflows/star_history.yml index 0734b27352..2c87130703 100644 --- a/.github/workflows/star_history.yml +++ b/.github/workflows/star_history.yml @@ -10,12 +10,11 @@ permissions: contents: write concurrency: - # Only one chart regeneration per branch at a time; a newer run on the same - # branch cancels an in-flight one so overlapping runs never conflict on - # note/star_history.svg during rebase. Scoped by ref so a manual run on - # another branch can't cancel the daily master update. + # Only one chart regeneration per branch at a time so overlapping runs never + # conflict on note/star_history.svg during rebase. Scoped by ref so a manual + # run on another branch can't cancel the daily master update. group: star-history-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} jobs: render: diff --git a/.github/workflows/telemetry-integration.yml b/.github/workflows/telemetry-integration.yml index 68dd2d1b79..12f99a204f 100644 --- a/.github/workflows/telemetry-integration.yml +++ b/.github/workflows/telemetry-integration.yml @@ -14,6 +14,10 @@ on: - 'weed/telemetry/**' - '.github/workflows/telemetry-integration.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/terraform_ci.yml b/.github/workflows/terraform_ci.yml index 1e905e6f0e..54ce78984d 100644 --- a/.github/workflows/terraform_ci.yml +++ b/.github/workflows/terraform_ci.yml @@ -8,6 +8,10 @@ on: branches: [ master ] paths: ['terraform/**', '.github/workflows/terraform_ci.yml'] +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/test-s3-over-https-using-awscli.yml b/.github/workflows/test-s3-over-https-using-awscli.yml index 903c3bf912..fff7b25143 100644 --- a/.github/workflows/test-s3-over-https-using-awscli.yml +++ b/.github/workflows/test-s3-over-https-using-awscli.yml @@ -27,6 +27,10 @@ defaults: run: working-directory: weed +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + jobs: awscli-tests: runs-on: ubuntu-latest diff --git a/.github/workflows/tls-rotation-tests.yml b/.github/workflows/tls-rotation-tests.yml index ce4ecc99d2..b0af119c13 100644 --- a/.github/workflows/tls-rotation-tests.yml +++ b/.github/workflows/tls-rotation-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/tls-rotation-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index 9e7f4e1c1f..eadec31a9e 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -11,8 +11,8 @@ on: - '.github/workflows/tus-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/tus-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read diff --git a/.github/workflows/vacuum-integration-tests.yml b/.github/workflows/vacuum-integration-tests.yml index e9f2a4ce13..f5b6259a83 100644 --- a/.github/workflows/vacuum-integration-tests.yml +++ b/.github/workflows/vacuum-integration-tests.yml @@ -18,6 +18,10 @@ on: - 'go.sum' - '.github/workflows/vacuum-integration-tests.yml' +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + permissions: contents: read diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index c1874fa198..9bea478215 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -21,8 +21,8 @@ on: - '.github/workflows/volume-server-integration-tests.yml' concurrency: - group: ${{ github.head_ref || github.ref }}/volume-server-integration-tests - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} permissions: contents: read From 9b59bc6b0df5a717b72976fe7262c5adb76c0f2a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 04:50:29 -0700 Subject: [PATCH 07/70] ci: build test binaries once, share them across shard jobs (#11981) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: build test binaries once, share them across shard jobs Each of the 6 rust-volume-go-tests shards, the rust-integration-tests job, and the 6 volume-server-integration-tests shards rebuilt the same weed and weed-volume binaries — 13 duplicate builds per run. A build-test-binaries / build-weed job now compiles them once under the existing cargo cache and publishes them as artifacts; the consumers download and chmod them instead of building. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: preserve binary paths through the artifact round-trip upload-artifact stores single files flat, so weed/weed would come back at the workspace root and the consumers' expected paths break. Pack the binaries in a tar before upload and extract on download. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: extract the shared test binaries in the rust go-test shards The download step leaves rust-test-binaries.tar at the workspace root; without untarring it the chmod fails on missing files and every shard fails before running a test. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/rust-volume-server-tests.yml | 126 ++++++++++-------- .../volume-server-integration-tests.yml | 53 ++++++-- 2 files changed, 119 insertions(+), 60 deletions(-) diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index b88a245876..7bcdbaf112 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -52,6 +52,63 @@ jobs: - 'seaweed-volume/**' - '.github/workflows/rust-volume-server-tests.yml' + build-test-binaries: + name: Build Test Binaries + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + + # cargo tracks its own inputs but not the runner's C toolchain, so a cached + # target/ can carry C objects built against a different glibc than we link against. + - name: Fingerprint build toolchain + id: toolchain + run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" + + - name: Cache cargo registry and target + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + seaweed-volume/target + key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} + restore-keys: | + rust-${{ steps.toolchain.outputs.fingerprint }}- + + - name: Build Go weed binary + run: | + cd weed + go build -tags 5BytesOffset -o weed . + chmod +x weed + ./weed version + + - name: Build Rust volume binary + run: cd seaweed-volume && cargo build --release + + - name: Package binaries + run: tar cf rust-test-binaries.tar weed/weed seaweed-volume/target/release/weed-volume + + - name: Upload test binaries + uses: actions/upload-artifact@v7 + with: + # archive layout mirrors the workspace: weed/weed and + # seaweed-volume/target/release/weed-volume + name: rust-test-binaries + if-no-files-found: error + retention-days: 1 + path: rust-test-binaries.tar + rust-unit-tests: name: Rust Unit Tests runs-on: ubuntu-22.04 @@ -154,6 +211,7 @@ jobs: name: Rust Integration Tests runs-on: ubuntu-22.04 timeout-minutes: 30 + needs: [build-test-binaries] steps: - name: Checkout code @@ -164,35 +222,16 @@ jobs: with: go-version-file: 'go.mod' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - # cargo tracks its own inputs but not the runner's C toolchain, so a cached - # target/ can carry C objects built against a different glibc than we link against. - - name: Fingerprint build toolchain - id: toolchain - run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - - - name: Cache cargo registry and target - uses: actions/cache@v6 + - name: Download test binaries + uses: actions/download-artifact@v7 with: - path: | - ~/.cargo/registry - ~/.cargo/git - seaweed-volume/target - key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} - restore-keys: | - rust-${{ steps.toolchain.outputs.fingerprint }}- + name: rust-test-binaries - - name: Build Go weed binary + - name: Restore binaries run: | - cd weed - go build -tags 5BytesOffset -o weed . - chmod +x weed - ./weed version - - - name: Build Rust volume binary - run: cd seaweed-volume && cargo build --release + tar xf rust-test-binaries.tar + chmod +x weed/weed seaweed-volume/target/release/weed-volume + ./weed/weed version - name: Run integration tests env: @@ -228,6 +267,7 @@ jobs: name: Go Tests with Rust Volume (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) runs-on: ubuntu-22.04 timeout-minutes: 45 + needs: [build-test-binaries] env: # Keep in step with the length of matrix.shard below. SHARD_COUNT: 3 @@ -246,35 +286,17 @@ jobs: with: go-version-file: 'go.mod' - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - - # cargo tracks its own inputs but not the runner's C toolchain, so a cached - # target/ can carry C objects built against a different glibc than we link against. - - name: Fingerprint build toolchain - id: toolchain - run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - - - name: Cache cargo registry and target - uses: actions/cache@v6 + - name: Download test binaries + uses: actions/download-artifact@v7 with: - path: | - ~/.cargo/registry - ~/.cargo/git - seaweed-volume/target - key: rust-${{ steps.toolchain.outputs.fingerprint }}-${{ hashFiles('seaweed-volume/Cargo.lock', 'seaweed-common/Cargo.lock') }} - restore-keys: | - rust-${{ steps.toolchain.outputs.fingerprint }}- + name: rust-test-binaries - - name: Build Go weed binary + - name: Restore binaries run: | - cd weed - go build -tags 5BytesOffset -o weed . - chmod +x weed - ./weed version - - - name: Build Rust volume binary - run: cd seaweed-volume && cargo build --release + tar xf rust-test-binaries.tar + chmod +x weed/weed seaweed-volume/target/release/weed-volume + ./weed/weed version + ./seaweed-volume/target/release/weed-volume --help >/dev/null || true # Dealing the listed tests out one by one keeps the shards even. Bucketing # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index 9bea478215..a1f98648e6 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -33,15 +33,10 @@ env: SHARD_COUNT: 3 jobs: - volume-server-integration-tests: - name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + build-weed: + name: Build SeaweedFS Binary runs-on: ubuntu-22.04 - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - test-type: [grpc, http] - shard: [1, 2, 3] + timeout-minutes: 15 steps: - name: Checkout code @@ -59,6 +54,48 @@ jobs: chmod +x weed ./weed version + - name: Package binary + run: tar cf weed-binary.tar weed/weed + + - name: Upload binary + uses: actions/upload-artifact@v7 + with: + name: weed-binary + if-no-files-found: error + retention-days: 1 + path: weed-binary.tar + + volume-server-integration-tests: + name: Volume Server Integration Tests (${{ matrix.test-type }} - Shard ${{ matrix.shard }}) + runs-on: ubuntu-22.04 + timeout-minutes: 45 + needs: [build-weed] + strategy: + fail-fast: false + matrix: + test-type: [grpc, http] + shard: [1, 2, 3] + + steps: + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + + - name: Download binary + uses: actions/download-artifact@v7 + with: + name: weed-binary + + - name: Restore binary + run: | + tar xf weed-binary.tar + chmod +x weed/weed + ./weed/weed version + # Dealing the listed tests out one by one keeps the shards even. Bucketing # them by first letter did not: names cluster, so ^Test[I-S] drew 50 of # the 114 grpc tests and ran nearly twice as long as the other two shards. From 728fc69976feb50abf6c7de1336303ac6dd24005 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 06:02:08 -0700 Subject: [PATCH 08/70] ci: cache docker build layers for container_dev (#11980) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci: cache docker build layers for container_dev The Go-build image rebuilds its layers on every run; type=gha backend caching reuses them across runs and platforms. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: pin container build to the pushed sha under layer caching Dockerfile.go_build clones master inside the image; a cache-from layer would otherwise keep serving the previous commit's clone and publish stale code under the dev tag. Passing BRANCH=github.sha makes the clone layer's key commit-specific while toolchain layers still hit. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: pass BRANCH via build-args so the per-commit cache key reaches buildx docker/build-push-action has no `args` input — the BRANCH override was silently dropped, leaving the build arg at its `master` default and the clone/checkout layers serving a stale commit under the gha cache. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/container_dev.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index f5d880a8ec..dd8c5843fc 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -197,6 +197,12 @@ jobs: platforms: linux/amd64, linux/arm64 tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + # Key the checkout layer on the pushed sha so cached layers can't + # serve stale code; the Dockerfile fetch fallback picks it up. + build-args: | + BRANCH=${{ github.sha }} - name: Sign if: github.event_name != 'pull_request' From 277c9809c86d991771380094ce783e526133d792 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 06:05:33 -0700 Subject: [PATCH 09/70] ci: skip setup-go cache on self-hosted runners (#11985) * ci: skip setup-go cache on self-hosted runners The module cache is already local on a persistent self-hosted runner; archiving ~/go/pkg/mod into cache.tzst at every job's end is pure cost and can stall (a ~4-minute zstdmt hang was observed on awscli-tests). runner.environment gates the cache to GitHub-hosted runners only. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: gate the remaining setup-go caches too The first pass only covered go-version-file steps; go-version steps in nine workflows (including awscli-tests' own file) kept uploading the cache archive on self-hosted runners. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../container_foundationdb_version.yml | 1 + .github/workflows/deploy_telemetry.yml | 1 + .github/workflows/e2e.yml | 1 + .github/workflows/ec-integration-tests.yml | 1 + .github/workflows/fuse-dlm-integration.yml | 1 + .github/workflows/fuse-failover.yml | 1 + .github/workflows/fuse-integration.yml | 1 + .github/workflows/fuse-p2p-integration.yml | 1 + .github/workflows/go.yml | 7 +++++++ .github/workflows/java_integration_tests.yml | 1 + .github/workflows/kafka-tests.yml | 3 +++ .github/workflows/kms-tests.yml | 2 ++ .github/workflows/master-cold-start-tests.yml | 1 + .../workflows/metadata-subscribe-tests.yml | 1 + .github/workflows/mount-benchmark.yml | 2 ++ .../workflows/mount-windows-conformance.yml | 1 + .github/workflows/mount-windows.yml | 1 + .github/workflows/multi-master-tests.yml | 1 + .github/workflows/performance.yml | 3 +++ .github/workflows/pjdfstest.yml | 1 + .github/workflows/plugin-workers.yml | 1 + .github/workflows/postgres-tests.yml | 1 + .../workflows/rust-volume-server-tests.yml | 2 ++ .github/workflows/s3-etag-acl-tests.yml | 1 + .../s3-example-integration-tests.yml | 1 + .github/workflows/s3-filer-group-tests.yml | 1 + .github/workflows/s3-go-tests.yml | 11 ++++++++++ .github/workflows/s3-iam-tests.yml | 4 ++++ .github/workflows/s3-keycloak-tests.yml | 1 + .../s3-mutation-regression-tests.yml | 3 +++ .github/workflows/s3-policy-tests.yml | 5 +++++ .../workflows/s3-proxy-signature-tests.yml | 1 + .github/workflows/s3-sdk-v2-routing-tests.yml | 1 + .github/workflows/s3-snowflake-tests.yml | 1 + .github/workflows/s3-spark-tests.yml | 1 + .github/workflows/s3-sse-tests.yml | 7 +++++++ .github/workflows/s3-tables-tests.yml | 20 +++++++++++++++++++ .github/workflows/s3tests.yml | 5 +++++ .github/workflows/samba-integration.yml | 1 + .github/workflows/sftp-tests.yml | 1 + .github/workflows/spark-integration-tests.yml | 1 + .github/workflows/telemetry-integration.yml | 1 + .github/workflows/terraform_ci.yml | 1 + .../test-s3-over-https-using-awscli.yml | 1 + .github/workflows/tls-rotation-tests.yml | 1 + .github/workflows/tus-tests.yml | 1 + .../workflows/vacuum-integration-tests.yml | 1 + .../volume-server-integration-tests.yml | 1 + 48 files changed, 109 insertions(+) diff --git a/.github/workflows/container_foundationdb_version.yml b/.github/workflows/container_foundationdb_version.yml index 6903dcf2c0..b976a9b245 100644 --- a/.github/workflows/container_foundationdb_version.yml +++ b/.github/workflows/container_foundationdb_version.yml @@ -68,6 +68,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} - name: Run FoundationDB tagged tests env: diff --git a/.github/workflows/deploy_telemetry.yml b/.github/workflows/deploy_telemetry.yml index e42cdea9ed..2f45c64176 100644 --- a/.github/workflows/deploy_telemetry.yml +++ b/.github/workflows/deploy_telemetry.yml @@ -27,6 +27,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'telemetry/server/go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build Telemetry Server if: github.event_name == 'workflow_dispatch' && inputs.deploy diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index ef5daf28bd..8ca45f0067 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -42,6 +42,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure Docker Hub mirror run: | diff --git a/.github/workflows/ec-integration-tests.yml b/.github/workflows/ec-integration-tests.yml index ec26d446eb..a1ddfa8e65 100644 --- a/.github/workflows/ec-integration-tests.yml +++ b/.github/workflows/ec-integration-tests.yml @@ -34,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/fuse-dlm-integration.yml b/.github/workflows/fuse-dlm-integration.yml index f4a7931d8b..869eae40ad 100644 --- a/.github/workflows/fuse-dlm-integration.yml +++ b/.github/workflows/fuse-dlm-integration.yml @@ -39,6 +39,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml index e73d588ba4..21646dbe17 100644 --- a/.github/workflows/fuse-failover.yml +++ b/.github/workflows/fuse-failover.yml @@ -46,6 +46,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-integration.yml b/.github/workflows/fuse-integration.yml index 8482eb287d..f45e3ed0a1 100644 --- a/.github/workflows/fuse-integration.yml +++ b/.github/workflows/fuse-integration.yml @@ -37,6 +37,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 8cb5f2eca7..78df490b43 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -45,6 +45,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Configure FUSE run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ffffaee697..0a06fbf6be 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -35,6 +35,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Get dependencies run: | cd weed; go get -v -t -d ./... @@ -57,6 +58,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Go Vet linux/386 (type-checks code and tests for 32-bit int overflows) run: | GOOS=linux GOARCH=386 go vet ./... 2>&1 | grep -v "MessageState contains sync.Mutex" | grep -v "IdentityAccessManagement contains sync.RWMutex" | tee vet-32bit-output.txt @@ -72,6 +74,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build run: cd weed; go build -tags "elastic gocdk sqlite ydb tarantool tikv rclone" -v . @@ -94,6 +97,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # Nothing else on a PR compiles these, so per-OS syscall constants creep # back into shared files unnoticed and only a release build catches it. - name: Cross-compile @@ -128,6 +132,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Test env: RUN_REDIS_TESTS: "1" @@ -143,6 +148,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # 386 test binaries run natively on the amd64 runner. This catches what vet # can't: unaligned 64-bit atomics and arithmetic that wraps at runtime. # -short skips the e2e suites already covered on amd64. @@ -159,6 +165,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # weed/kms/azure is excluded from every default build, so without this # nothing compiles it and tag-gated code silently rots. - name: Build and test with azurekms diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml index 9dc3461c82..aa8a93c9e4 100644 --- a/.github/workflows/java_integration_tests.yml +++ b/.github/workflows/java_integration_tests.yml @@ -35,6 +35,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Java diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index c0e15cee89..8fb5cb5a36 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -53,6 +53,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go @@ -98,6 +99,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go @@ -805,6 +807,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/kms-tests.yml b/.github/workflows/kms-tests.yml index 6e020f0dea..6b77a896f1 100644 --- a/.github/workflows/kms-tests.yml +++ b/.github/workflows/kms-tests.yml @@ -43,6 +43,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -88,6 +89,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/master-cold-start-tests.yml b/.github/workflows/master-cold-start-tests.yml index bd983c7c84..bc96b833bf 100644 --- a/.github/workflows/master-cold-start-tests.yml +++ b/.github/workflows/master-cold-start-tests.yml @@ -42,6 +42,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed binary run: | diff --git a/.github/workflows/metadata-subscribe-tests.yml b/.github/workflows/metadata-subscribe-tests.yml index e674986b0d..b0bb96bc25 100644 --- a/.github/workflows/metadata-subscribe-tests.yml +++ b/.github/workflows/metadata-subscribe-tests.yml @@ -48,6 +48,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/mount-benchmark.yml b/.github/workflows/mount-benchmark.yml index 52c7dcb49d..206e082243 100644 --- a/.github/workflows/mount-benchmark.yml +++ b/.github/workflows/mount-benchmark.yml @@ -32,6 +32,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install WinFsp and rclone run: choco install winfsp rclone -y --no-progress @@ -150,6 +151,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Repair the fusermount3 setuid bit uses: ./.github/actions/fix-fusermount-setuid diff --git a/.github/workflows/mount-windows-conformance.yml b/.github/workflows/mount-windows-conformance.yml index c9aab26a02..c58d063575 100644 --- a/.github/workflows/mount-windows-conformance.yml +++ b/.github/workflows/mount-windows-conformance.yml @@ -49,6 +49,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but # not to build. diff --git a/.github/workflows/mount-windows.yml b/.github/workflows/mount-windows.yml index 18cac2773a..7954176ce7 100644 --- a/.github/workflows/mount-windows.yml +++ b/.github/workflows/mount-windows.yml @@ -49,6 +49,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} # cgofuse loads winfsp-x64.dll at run time, so WinFsp is needed here but # not to build. diff --git a/.github/workflows/multi-master-tests.yml b/.github/workflows/multi-master-tests.yml index b42d507f09..af282120b8 100644 --- a/.github/workflows/multi-master-tests.yml +++ b/.github/workflows/multi-master-tests.yml @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 8dd438382c..599c01770c 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -67,6 +67,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed run: go build -o weed_bin ./weed @@ -175,6 +176,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install Rust toolchain if: matrix.impl == 'rust' @@ -327,6 +329,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install Rust toolchain if: matrix.impl == 'rust' diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml index 80845380ca..878f787f4f 100644 --- a/.github/workflows/pjdfstest.yml +++ b/.github/workflows/pjdfstest.yml @@ -38,6 +38,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 diff --git a/.github/workflows/plugin-workers.yml b/.github/workflows/plugin-workers.yml index 94fb1fbf60..55d3f56478 100644 --- a/.github/workflows/plugin-workers.yml +++ b/.github/workflows/plugin-workers.yml @@ -45,6 +45,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml index d3eff2fe15..ed26b591b9 100644 --- a/.github/workflows/postgres-tests.yml +++ b/.github/workflows/postgres-tests.yml @@ -41,6 +41,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index 7bcdbaf112..eb5db5c739 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -221,6 +221,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Download test binaries uses: actions/download-artifact@v7 @@ -285,6 +286,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Download test binaries uses: actions/download-artifact@v7 diff --git a/.github/workflows/s3-etag-acl-tests.yml b/.github/workflows/s3-etag-acl-tests.yml index 13fb6cffb9..5f8754a372 100644 --- a/.github/workflows/s3-etag-acl-tests.yml +++ b/.github/workflows/s3-etag-acl-tests.yml @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/s3-example-integration-tests.yml b/.github/workflows/s3-example-integration-tests.yml index 1765b2f818..d0855c65ca 100644 --- a/.github/workflows/s3-example-integration-tests.yml +++ b/.github/workflows/s3-example-integration-tests.yml @@ -33,6 +33,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Build SeaweedFS diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml index 2fcfc8e692..18f64fd78c 100644 --- a/.github/workflows/s3-filer-group-tests.yml +++ b/.github/workflows/s3-filer-group-tests.yml @@ -36,6 +36,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 168d75efc3..6f5de41974 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -39,6 +39,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -103,6 +104,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -151,6 +153,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -202,6 +205,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -293,6 +297,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -338,6 +343,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -396,6 +402,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -460,6 +467,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -509,6 +517,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -577,6 +586,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -632,6 +642,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml index 5bf89d7e70..ee12afd842 100644 --- a/.github/workflows/s3-iam-tests.yml +++ b/.github/workflows/s3-iam-tests.yml @@ -45,6 +45,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Get dependencies @@ -94,6 +95,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -208,6 +210,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -268,6 +271,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml index a5a34d45f3..d893b1063b 100644 --- a/.github/workflows/s3-keycloak-tests.yml +++ b/.github/workflows/s3-keycloak-tests.yml @@ -41,6 +41,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-mutation-regression-tests.yml b/.github/workflows/s3-mutation-regression-tests.yml index 0f30ef4edb..d352215176 100644 --- a/.github/workflows/s3-mutation-regression-tests.yml +++ b/.github/workflows/s3-mutation-regression-tests.yml @@ -32,6 +32,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Run S3 versioning regression tests timeout-minutes: 20 @@ -85,6 +86,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Run S3 delete regression tests timeout-minutes: 15 @@ -123,6 +125,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml index 99c1949b18..caa9f9de6f 100644 --- a/.github/workflows/s3-policy-tests.yml +++ b/.github/workflows/s3-policy-tests.yml @@ -49,6 +49,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Get dependencies @@ -92,6 +93,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -201,6 +203,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -319,6 +322,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -406,6 +410,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-proxy-signature-tests.yml b/.github/workflows/s3-proxy-signature-tests.yml index 40756e0982..ded2068737 100644 --- a/.github/workflows/s3-proxy-signature-tests.yml +++ b/.github/workflows/s3-proxy-signature-tests.yml @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror diff --git a/.github/workflows/s3-sdk-v2-routing-tests.yml b/.github/workflows/s3-sdk-v2-routing-tests.yml index 477c5ef7fd..65aac2004a 100644 --- a/.github/workflows/s3-sdk-v2-routing-tests.yml +++ b/.github/workflows/s3-sdk-v2-routing-tests.yml @@ -34,6 +34,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/s3-snowflake-tests.yml b/.github/workflows/s3-snowflake-tests.yml index ec3ec6a32a..3993a112a0 100644 --- a/.github/workflows/s3-snowflake-tests.yml +++ b/.github/workflows/s3-snowflake-tests.yml @@ -60,6 +60,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Java diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index b6e7580a8c..39ce8d9991 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -31,6 +31,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index 9dfe3f8758..4c44e42448 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -50,6 +50,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -118,6 +119,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -166,6 +168,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -215,6 +218,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -264,6 +268,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -315,6 +320,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS @@ -363,6 +369,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index 275098d960..64aa3d7fbc 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -34,6 +34,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -94,6 +95,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -155,6 +157,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -225,6 +228,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -298,6 +302,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -381,6 +386,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -468,6 +474,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -567,6 +574,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -635,6 +643,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -705,6 +714,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -776,6 +786,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -870,6 +881,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run go mod tidy @@ -930,6 +942,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1003,6 +1016,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1073,6 +1087,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1145,6 +1160,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1220,6 +1236,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Configure Docker Hub mirror @@ -1290,6 +1307,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Verify S3 Tables Package Builds @@ -1355,6 +1373,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Check Go Format @@ -1406,6 +1425,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run Go Vet diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 353009c4ce..26f7a83553 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -44,6 +44,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -396,6 +397,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -566,6 +568,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python @@ -691,6 +694,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Run SeaweedFS Custom S3 Copy tests @@ -793,6 +797,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Set up Python diff --git a/.github/workflows/samba-integration.yml b/.github/workflows/samba-integration.yml index 082ec5a6df..f37020090a 100644 --- a/.github/workflows/samba-integration.yml +++ b/.github/workflows/samba-integration.yml @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 diff --git a/.github/workflows/sftp-tests.yml b/.github/workflows/sftp-tests.yml index 9782d4e240..0930b06b92 100644 --- a/.github/workflows/sftp-tests.yml +++ b/.github/workflows/sftp-tests.yml @@ -40,6 +40,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml index 20c2830e34..ee211a1fb4 100644 --- a/.github/workflows/spark-integration-tests.yml +++ b/.github/workflows/spark-integration-tests.yml @@ -48,6 +48,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS binary run: | diff --git a/.github/workflows/telemetry-integration.yml b/.github/workflows/telemetry-integration.yml index 12f99a204f..b7aa690fb6 100644 --- a/.github/workflows/telemetry-integration.yml +++ b/.github/workflows/telemetry-integration.yml @@ -34,6 +34,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build telemetry server run: cd telemetry/server && go build -o telemetry-server . diff --git a/.github/workflows/terraform_ci.yml b/.github/workflows/terraform_ci.yml index 54ce78984d..b2bcca7500 100644 --- a/.github/workflows/terraform_ci.yml +++ b/.github/workflows/terraform_ci.yml @@ -66,6 +66,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: go.mod + cache: ${{ runner.environment == 'github-hosted' }} - name: Build weed run: go build -o "$RUNNER_TEMP/weed" ./weed diff --git a/.github/workflows/test-s3-over-https-using-awscli.yml b/.github/workflows/test-s3-over-https-using-awscli.yml index fff7b25143..8ccb35eeb7 100644 --- a/.github/workflows/test-s3-over-https-using-awscli.yml +++ b/.github/workflows/test-s3-over-https-using-awscli.yml @@ -41,6 +41,7 @@ jobs: - uses: actions/setup-go@v7 with: go-version: ^1.26 + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/tls-rotation-tests.yml b/.github/workflows/tls-rotation-tests.yml index b0af119c13..669ca4794e 100644 --- a/.github/workflows/tls-rotation-tests.yml +++ b/.github/workflows/tls-rotation-tests.yml @@ -34,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index eadec31a9e..699c36b0b5 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -35,6 +35,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} id: go - name: Install SeaweedFS diff --git a/.github/workflows/vacuum-integration-tests.yml b/.github/workflows/vacuum-integration-tests.yml index f5b6259a83..93eefec26e 100644 --- a/.github/workflows/vacuum-integration-tests.yml +++ b/.github/workflows/vacuum-integration-tests.yml @@ -34,6 +34,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v7 with: + cache: ${{ runner.environment == 'github-hosted' }} go-version: ^1.26 id: go diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index a1f98648e6..fcaf1ce976 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -46,6 +46,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Build SeaweedFS binary run: | From 384523761bf80616827c6a8fc92de195d2041167 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 06:19:56 -0700 Subject: [PATCH 10/70] ci: skip actions/cache on self-hosted runners (#11986) cargo registry/target and docker-image caches are already local on a persistent self-hosted runner; the save/restore pass is pure overhead (the setup-go cache-save stall on awscli-tests was the same pattern). Gate every actions/cache step on runner.environment == 'github-hosted'. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/container_dev.yml | 1 + .github/workflows/container_latest.yml | 1 + .github/workflows/container_release_unified.yml | 1 + .github/workflows/e2e.yml | 1 + .github/workflows/performance.yml | 4 ++-- .github/workflows/postgres-tests.yml | 1 + .github/workflows/rust-volume-server-tests.yml | 3 +++ .github/workflows/rust-worker-tests.yml | 1 + .github/workflows/rust_binaries_dev.yml | 2 ++ .github/workflows/rust_binaries_release.yml | 4 ++++ .github/workflows/s3-tables-tests.yml | 1 + 11 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index dd8c5843fc..2c30f861a2 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -55,6 +55,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 3d99baaf24..0dc49db29c 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -99,6 +99,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 095524ce34..6ac4ce06db 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -78,6 +78,7 @@ jobs: echo "CFLAGS_aarch64_unknown_linux_musl=-U_FORTIFY_SOURCE" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8ca45f0067..363627df27 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -53,6 +53,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Cache Docker layers + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/.buildx-cache diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 599c01770c..0d22915b22 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -183,7 +183,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target - if: matrix.impl == 'rust' + if: matrix.impl == 'rust' && runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -336,7 +336,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target - if: matrix.impl == 'rust' + if: matrix.impl == 'rust' && runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/postgres-tests.yml b/.github/workflows/postgres-tests.yml index ed26b591b9..bbcef063bf 100644 --- a/.github/workflows/postgres-tests.yml +++ b/.github/workflows/postgres-tests.yml @@ -57,6 +57,7 @@ jobs: uses: docker/setup-buildx-action@v4 - name: Cache Docker layers + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/.buildx-cache diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index eb5db5c739..84b80de66d 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -76,6 +76,7 @@ jobs: run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -128,6 +129,7 @@ jobs: run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -191,6 +193,7 @@ jobs: run: echo "fingerprint=windows-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/rust-worker-tests.yml b/.github/workflows/rust-worker-tests.yml index 66510cf101..7bfaecb695 100644 --- a/.github/workflows/rust-worker-tests.yml +++ b/.github/workflows/rust-worker-tests.yml @@ -45,6 +45,7 @@ jobs: run: echo "fingerprint=$(getconf GNU_LIBC_VERSION | tr ' ' '-')-rustc-$(rustc -V | awk '{print $2}')" >> "$GITHUB_OUTPUT" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/rust_binaries_dev.yml b/.github/workflows/rust_binaries_dev.yml index 563022d028..e49abdbbd8 100644 --- a/.github/workflows/rust_binaries_dev.yml +++ b/.github/workflows/rust_binaries_dev.yml @@ -46,6 +46,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -114,6 +115,7 @@ jobs: targets: ${{ matrix.target }} - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/rust_binaries_release.yml b/.github/workflows/rust_binaries_release.yml index a5a8b61455..f9446a917a 100644 --- a/.github/workflows/rust_binaries_release.yml +++ b/.github/workflows/rust_binaries_release.yml @@ -48,6 +48,7 @@ jobs: echo "OPENSSL_LIB_DIR=/usr/lib/aarch64-linux-gnu" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -153,6 +154,7 @@ jobs: echo "CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc" >> "$GITHUB_ENV" - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -230,6 +232,7 @@ jobs: targets: ${{ matrix.target }} - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | @@ -305,6 +308,7 @@ jobs: uses: dtolnay/rust-toolchain@stable - name: Cache cargo registry and target + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: | diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index 64aa3d7fbc..baa50a4cb9 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -800,6 +800,7 @@ jobs: - name: Restore python:3 image cache id: python-image + if: runner.environment == 'github-hosted' uses: actions/cache@v6 with: path: /tmp/python3-image.tar From c4002fafc79ebc497377a9b08d748c4f14877d08 Mon Sep 17 00:00:00 2001 From: bati-valensas <150924329+bati-valensas@users.noreply.github.com> Date: Fri, 23 Oct 2026 16:43:35 +0300 Subject: [PATCH 11/70] helm: fix bucket hook treating an existing bucket as missing (#11982) * helm: fix bucket hook treating an existing bucket as missing The skip check pipes awk into `grep -q`, which exits on the first match and can leave awk writing into a closed pipe. awk then dies of SIGPIPE and `set -o pipefail` makes the pipeline report that, so a bucket that exists reads as missing: the hook runs `s3.bucket.create`, it errors with "already exists", and `sh -ec` aborts the hook - failing the release. Match in a single awk instead, which reads to EOF and cannot be killed by a closing reader. * helm: force string compare in bucket hook existence check --------- Co-authored-by: Chris Lu --- .../seaweedfs/templates/shared/post-install-bucket-hook.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml index 11ea7feff6..3952c73fbf 100644 --- a/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml +++ b/k8s/charts/seaweedfs/templates/shared/post-install-bucket-hook.yaml @@ -172,7 +172,8 @@ spec: {{- $bucketName := .name }} {{- $bucketLock := or .lock .objectLock .withLock }} bucket_list=$(/bin/echo 's3.bucket.list' | /usr/bin/weed shell) || { echo "Error listing s3 buckets"; exit 1; } - if echo "$bucket_list" | awk '{print $1}' | grep -Fxq "{{ $bucketName }}"; then + {{- /* One awk: grep -q exits early, awk takes SIGPIPE, pipefail fails the test. */}} + if printf '%s\n' "$bucket_list" | awk -v bucket="{{ $bucketName }}" '$1 "" == bucket { found = 1 } END { exit !found }'; then echo "Bucket '{{ $bucketName }}' already exists, skipping creation." else echo "Creating bucket '{{ $bucketName }}'..." From 2c8079625c7830b1071d22fde3d4bedcaf05d45d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 08:08:49 -0700 Subject: [PATCH 12/70] test: stop EC integration tests from pinning default master port 9333 (#11988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EC integration tests failed in CI with "raft.Server: Not current leader" on every assign for tens of seconds — longer than any goraft election takes. A stale weed process on the self-hosted runner was still holding 9333, so each test's fresh master died at bind time and the tests kept talking to a raft non-leader. Only the 9333 cluster (startSeaweedFSCluster) failed; tests on 9334-9338 all passed. Allocate mini-safe ports (http + grpc+10000 both verified free) for the master and volume servers instead of hardcoding 9333/8080-8085, wait for the master to actually report raft leadership via /cluster/status before issuing raft-gated RPCs, and make waitForServer a real TCP dial — grpc.NewClient never connects, so the old check always passed instantly. --- test/erasure_coding/ec_integration_test.go | 91 ++++++++++++++++------ 1 file changed, 68 insertions(+), 23 deletions(-) diff --git a/test/erasure_coding/ec_integration_test.go b/test/erasure_coding/ec_integration_test.go index f3a1461336..33be5a46b8 100644 --- a/test/erasure_coding/ec_integration_test.go +++ b/test/erasure_coding/ec_integration_test.go @@ -3,9 +3,12 @@ package erasure_coding import ( "bytes" "context" + "encoding/json" "fmt" "io" "math" + "net" + "net/http" "os" "os/exec" "path/filepath" @@ -13,6 +16,7 @@ import ( "testing" "time" + "github.com/seaweedfs/seaweedfs/test/testutil" "github.com/seaweedfs/seaweedfs/weed/operation" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" @@ -47,17 +51,15 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8081", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8082", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8083", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8084", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8085", 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + for _, addr := range cluster.volumeAddrs { + require.NoError(t, waitForServer(addr, 30*time.Second)) + } + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -71,7 +73,7 @@ func TestECEncodingVolumeLocationTimingBug(t *testing.T) { // Upload some test data to create volumes testData := []byte("This is test data for EC encoding integration test") - volumeId, err := uploadTestData(testData, "127.0.0.1:9333") + volumeId, err := uploadTestData(testData, cluster.masterAddr) require.NoError(t, err) t.Logf("Created volume %d with test data", volumeId) @@ -257,12 +259,13 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - require.NoError(t, waitForServer("127.0.0.1:8080", 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + require.NoError(t, waitForServer(cluster.volumeAddrs[0], 30*time.Second)) + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -276,7 +279,7 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { // Upload test data testData := []byte("Race condition test data") - volumeId, err := uploadTestData(testData, "127.0.0.1:9333") + volumeId, err := uploadTestData(testData, cluster.masterAddr) require.NoError(t, err) t.Logf("Created volume %d for race condition test", volumeId) @@ -366,6 +369,8 @@ func TestECEncodingMasterTimingRaceCondition(t *testing.T) { type TestCluster struct { masterCmd *exec.Cmd volumeServers []*exec.Cmd + masterAddr string + volumeAddrs []string } func (c *TestCluster) Stop() { @@ -402,9 +407,23 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e securityToml := filepath.Join(dataDir, "security.toml") os.WriteFile(securityToml, []byte("# Empty security config for testing\n"), 0644) + // Allocate ports instead of using the defaults: on a shared or reused + // runner a stale weed process can still hold 9333/808x, which kills the + // freshly started master at bind time and leaves the test talking to a + // raft follower that answers every assign with "not current leader". + // Mini-safe ports guarantee each HTTP port's derived gRPC port + // (port + 10000) is free as well. + ports, err := testutil.AllocateMiniPorts(7) + if err != nil { + return nil, fmt.Errorf("failed to allocate ports: %v", err) + } + masterPort := ports[0] + masterAddr := fmt.Sprintf("127.0.0.1:%d", masterPort) + cluster.masterAddr = masterAddr + // Start master server masterCmd := exec.CommandContext(ctx, weedBinary, "master", - "-port", "9333", + "-port", fmt.Sprintf("%d", masterPort), "-mdir", masterDir, "-volumeSizeLimitMB", "10", // Small volumes for testing "-ip", "127.0.0.1", @@ -432,13 +451,14 @@ func startSeaweedFSCluster(ctx context.Context, dataDir string) (*TestCluster, e volumeDir := filepath.Join(dataDir, fmt.Sprintf("volume%d", i)) os.MkdirAll(volumeDir, 0755) - port := fmt.Sprintf("808%d", i) + port := fmt.Sprintf("%d", ports[1+i]) + cluster.volumeAddrs = append(cluster.volumeAddrs, fmt.Sprintf("127.0.0.1:%d", ports[1+i])) rack := fmt.Sprintf("rack%d", i) volumeCmd := exec.CommandContext(ctx, weedBinary, "volume", "-port", port, "-dir", volumeDir, "-max", "10", - "-master", "127.0.0.1:9333", + "-master", masterAddr, "-volume.allowUntrustedRemoteEndpoints", "-ip", "127.0.0.1", "-dataCenter", "dc1", @@ -498,7 +518,8 @@ func findWeedBinary() string { func waitForServer(address string, timeout time.Duration) error { start := time.Now() for time.Since(start) < timeout { - if conn, err := grpc.NewClient(address, grpc.WithInsecure()); err == nil { + conn, err := net.DialTimeout("tcp", address, time.Second) + if err == nil { conn.Close() return nil } @@ -507,6 +528,29 @@ func waitForServer(address string, timeout time.Duration) error { return fmt.Errorf("timeout waiting for server %s", address) } +// waitForMasterLeader polls the master's /cluster/status endpoint until it +// reports itself as the raft leader. A goraft master can take up to the +// configured election timeout (10s by default) before it is ready to serve +// raft-gated requests like volume assign, so a bare port check is not enough. +func waitForMasterLeader(masterAddr string, timeout time.Duration) error { + start := time.Now() + for time.Since(start) < timeout { + resp, err := http.Get(fmt.Sprintf("http://%s/cluster/status", masterAddr)) + if err == nil { + var status struct { + IsLeader bool `json:"IsLeader"` + } + err = json.NewDecoder(resp.Body).Decode(&status) + resp.Body.Close() + if err == nil && status.IsLeader { + return nil + } + } + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("timeout waiting for master %s to become raft leader", masterAddr) +} + func uploadTestData(data []byte, masterAddress string) (needle.VolumeId, error) { // Upload data to get a file ID assignResult, err := operation.Assign(context.Background(), func(ctx context.Context) pb.ServerAddress { @@ -2209,14 +2253,15 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { defer cluster.Stop() // Wait for servers to be ready - require.NoError(t, waitForServer("127.0.0.1:9333", 30*time.Second)) - for i := 0; i < 6; i++ { - require.NoError(t, waitForServer(fmt.Sprintf("127.0.0.1:%d", 8080+i), 30*time.Second)) + require.NoError(t, waitForServer(cluster.masterAddr, 30*time.Second)) + for _, addr := range cluster.volumeAddrs { + require.NoError(t, waitForServer(addr, 30*time.Second)) } + require.NoError(t, waitForMasterLeader(cluster.masterAddr, 30*time.Second)) // Create command environment options := &shell.ShellOptions{ - Masters: stringPtr("127.0.0.1:9333"), + Masters: stringPtr(cluster.masterAddr), GrpcDialOption: grpc.WithInsecure(), FilerGroup: stringPtr("default"), } @@ -2240,7 +2285,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { var volumeId needle.VolumeId var uploadErr error for retry := 0; retry < 5; retry++ { - volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication) + volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication) if uploadErr == nil { break } @@ -2290,7 +2335,7 @@ func TestECEncodeReplicatedVolumeSync(t *testing.T) { var volumeId needle.VolumeId var uploadErr error for retry := 0; retry < 5; retry++ { - volumeId, uploadErr = uploadTestDataWithReplication(t, "127.0.0.1:9333", testCollection, testReplication) + volumeId, uploadErr = uploadTestDataWithReplication(t, cluster.masterAddr, testCollection, testReplication) if uploadErr == nil { break } From 34f06ee92f22c8e354184108d304517400097ba0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 08:10:31 -0700 Subject: [PATCH 13/70] test/fuse: fix cluster startup in the fuse test framework (#11977) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test/fuse: fix cluster startup in the fuse test framework weed volume Fatalfs when -dir does not exist, so every cluster setup failed its volume wait after 30s. The mount also exited on the unknown -cacheSizeMB flag (it is -cacheCapacityMB now). * test/fuse: manage cluster processes and allocate free service ports Each service now runs through a managed process that captures its log and reports an early exit from the port wait, instead of a bare 30s timeout. Ports are allocated free pairs (http + grpc sibling) rather than fixed 19333/18080/18888 so the framework no longer collides with weed mini's fixed ports or another framework instance. * test/fuse: bail out of the mount wait when the mount process exits early The port waits already report an early exit with the captured log tail; the mount wait still polled the device id for the full timeout, so a mount that died at startup (bad flag, missing filer) surfaced as a bare 30s timeout with no diagnosis. Check the managed process the same way and include its log tail. * test/fuse: deduplicate service ports handed out by freeServicePort A probed port is released before the service binds it, so the three sequential allocations for master, volume and filer could all return the same still-unbound port and leave later services dying at startup. Track handed-out ports so each allocation is unique. * test/fuse: allocate service ports in Setup after the external-mount branch * test/fuse: verify service readiness through its own /healthz endpoint * test/fuse: bound the readiness probe so a silent listener cannot hang setup * test/fuse: only accept a ready response while the service process still runs * test/fuse: include the service log tail on readiness timeout A startup that stalls instead of exiting still failed with a bare "not ready within timeout" — the same opaque message that made the original flake hard to diagnose. The process-exit path already tails the captured service log; the timeout path now does too. * test/fuse: report pjdfstest category results without failing the suite Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: allow allow_other mounts in posix-compliance The job mounts with -allowOthers but never sets user_allow_other in /etc/fuse.conf, so fusermount rejects the option and the mount exits before the suite runs. The other FUSE workflows set it; do the same. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: keep the standalone-framework service logs on failure Cleanup removed the temp dir holding master/volume/filer/mount logs, so a CI failure left nothing to diagnose. Dump the tails on failure and copy them under /tmp/seaweedfs-fuse-logs/standalone/ for the artifact upload, next to the mini framework's logs. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: gate setup on a real write through the mount Mount readiness only proves the daemon is serving; a fresh cluster still cannot assign a file id until a volume heartbeats in and a grow round finishes, so the first ~10-30s of writes surface as EIO on close. Probe with a real write/read/remove before handing the mount to the suite. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: gate the mini framework setup on a real write too Same cold-start window as the standalone framework: mini's volume has to register and grow before a write can be assigned. Keep the two frameworks consistent so later suites do not flake on the same stall. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: save each framework's CI logs under its test name Fixed-name mini.log/mount.log in /tmp/seaweedfs-fuse-logs let a later framework instance overwrite an earlier test's logs, so an uploaded artifact could describe a passing run instead of the failure under investigation. Apply to both framework copies since they share the directory. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: bound each waitForWritable probe attempt A wedged mount can hold a file op open past the 90s deadline, so the loop could not check the deadline or a mount exit and Setup hung until the overall test timeout. Each attempt now runs off the loop and is abandoned when the remaining time runs out. Same fix in both framework copies. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test/fuse: keep one write probe in flight until it finishes or the deadline --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf --- test/fuse_integration/framework.go | 131 ++++++++++++++++--- test/fuse_integration/framework_test.go | 83 +++++++++++- test/fuse_integration/posix_external_test.go | 7 +- 3 files changed, 198 insertions(+), 23 deletions(-) diff --git a/test/fuse_integration/framework.go b/test/fuse_integration/framework.go index 47280e652c..a6d002eeef 100644 --- a/test/fuse_integration/framework.go +++ b/test/fuse_integration/framework.go @@ -4,10 +4,12 @@ import ( "fmt" "io/fs" "net" + "net/http" "os" "os/exec" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -241,6 +243,14 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { return fmt.Errorf("FUSE mount not ready: %v", err) } + // A mounted filesystem still cannot write until a volume registers and a + // grow round finishes, so gate on a real write instead of letting the + // first test hits ride out the startup window as EIO on close. + if err := f.waitForWritable(90 * time.Second); err != nil { + f.dumpLog("mount") + return fmt.Errorf("FUSE mount not writable: %v", err) + } + f.isSetup = true return nil } @@ -298,9 +308,10 @@ func (f *FuseTestFramework) GetFilerAddr() string { // managedProcess is a started weed sub-command whose exit is watched, so a wait // for it to come up ends the moment it dies instead of burning its full timeout. type managedProcess struct { - cmd *exec.Cmd - done chan struct{} - err error // exit error, read only after done is closed + cmd *exec.Cmd + done chan struct{} + err error // exit error, read only after done is closed + logPath string } // exited returns the exit error once the process is gone, nil while it runs. @@ -332,7 +343,8 @@ func (p *managedProcess) stop() { // startProcess is a helper that starts a weed sub-command with output captured // to a log file in f.logDir. func (f *FuseTestFramework) startProcess(name string, args []string) (*managedProcess, error) { - logFile, err := os.Create(filepath.Join(f.logDir, name+".log")) + logPath := filepath.Join(f.logDir, name+".log") + logFile, err := os.Create(logPath) if err != nil { return nil, fmt.Errorf("create log file: %v", err) } @@ -347,7 +359,7 @@ func (f *FuseTestFramework) startProcess(name string, args []string) (*managedPr // Close the file handle — the child process inherited it. logFile.Close() - p := &managedProcess{cmd: cmd, done: make(chan struct{})} + p := &managedProcess{cmd: cmd, done: make(chan struct{}), logPath: logPath} go func() { p.err = cmd.Wait() close(p.done) @@ -355,6 +367,20 @@ func (f *FuseTestFramework) startProcess(name string, args []string) (*managedPr return p, nil } +// logTail returns the last bytes of a captured process log so a startup +// failure message carries the fatal line instead of a bare timeout. +func logTail(path string) string { + data, err := os.ReadFile(path) + if err != nil || len(data) == 0 { + return "" + } + const maxTail = 2 * 1024 + if len(data) > maxTail { + data = data[len(data)-maxTail:] + } + return string(data) +} + // dumpLog prints the last lines of a process log file to the test output // for debugging when a service fails to start or a test fails. func (f *FuseTestFramework) dumpLog(name string) { @@ -371,11 +397,16 @@ func (f *FuseTestFramework) dumpLog(name string) { f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data)) } -// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/ -// so the CI workflow can upload them as artifacts. +// copyLogsForCI copies SeaweedFS process logs under a per-test directory in +// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts. +// Each framework instance gets its own directory: every test in the suite +// builds one, and fixed filenames would let a later passing test overwrite +// the logs of an earlier failing one. func (f *FuseTestFramework) copyLogsForCI() { - ciLogDir := "/tmp/seaweedfs-fuse-logs" - os.MkdirAll(ciLogDir, 0755) + ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name())) + if err := os.MkdirAll(ciLogDir, 0755); err != nil { + return + } for _, name := range []string{"mini", "mount"} { src := filepath.Join(f.logDir, name+".log") data, err := os.ReadFile(src) @@ -464,21 +495,25 @@ func (f *FuseTestFramework) unmountFuse() error { return nil } -// waitForService waits for a service to be available +// waitForService waits until the weed process answers its own /healthz, so a +// foreign listener on the port cannot pass for readiness. func (f *FuseTestFramework) waitForService(proc *managedProcess, addr string, timeout time.Duration) error { deadline := time.Now().Add(timeout) + client := &http.Client{Timeout: time.Second} for time.Now().Before(deadline) { - conn, err := net.DialTimeout("tcp", addr, 1*time.Second) + resp, err := client.Get("http://" + addr + "/healthz") if err == nil { - conn.Close() - return nil + resp.Body.Close() + if resp.StatusCode == http.StatusOK && proc.exited() == nil { + return nil + } } if exitErr := proc.exited(); exitErr != nil { - return fmt.Errorf("process exited (%v) before %s accepted connections", exitErr, addr) + return fmt.Errorf("process exited (%v) before %s accepted connections: %s", exitErr, addr, logTail(proc.logPath)) } time.Sleep(100 * time.Millisecond) } - return fmt.Errorf("service at %s not ready within timeout", addr) + return fmt.Errorf("service at %s not ready within timeout: %s", addr, logTail(proc.logPath)) } // waitForMount waits for the FUSE mount to be ready. A stat/ReadDir probe @@ -501,11 +536,73 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { // A mount that cannot mount at all (no /dev/fuse, fusermount not setuid) // dies within a second; reporting that beats waiting out the timeout. if exitErr := f.mountProcess.exited(); exitErr != nil { - return fmt.Errorf("mount process exited (%v)", exitErr) + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) } time.Sleep(100 * time.Millisecond) } - return fmt.Errorf("mount point not ready within timeout") + return fmt.Errorf("mount point not ready within timeout: %s", logTail(f.mountProcess.logPath)) +} + +// waitForWritable probes the mount with a real write+read+remove until one +// succeeds, covering the window where a fresh cluster cannot yet assign a +// file id (volume not registered, grow pending, leader settle). Each attempt +// runs off the loop because a wedged mount can hold a file op open, and one +// attempt stays in flight until it finishes or the overall deadline passes. +func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error { + probe := filepath.Join(f.mountPoint, ".framework_write_probe") + payload := []byte("probe") + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) + } + result := make(chan error, 1) + go func() { + if err := os.WriteFile(probe, payload, 0644); err != nil { + result <- err + return + } + data, err := os.ReadFile(probe) + if err != nil { + result <- err + return + } + if len(data) != len(payload) { + result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload)) + return + } + result <- os.Remove(probe) + }() + tick := time.NewTicker(500 * time.Millisecond) + attemptOpen := true + for attemptOpen { + select { + case err := <-result: + attemptOpen = false + if err == nil { + tick.Stop() + return nil + } + lastErr = err + case <-tick.C: + if exitErr := f.mountProcess.exited(); exitErr != nil { + tick.Stop() + return fmt.Errorf("mount process exited (%v): %s", exitErr, logTail(f.mountProcess.logPath)) + } + if !time.Now().Before(deadline) { + tick.Stop() + if lastErr == nil { + return fmt.Errorf("write probe did not complete within timeout") + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) + } + } + } + tick.Stop() + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) } // fuseAvailable reports whether the host has a FUSE device node to mount diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index 23af7e9dd2..f045fa157b 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strconv" + "strings" "syscall" "testing" "time" @@ -168,6 +169,13 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { return fmt.Errorf("FUSE mount not ready: %v", err) } + // Gate on a real write: a fresh mini still cannot assign a file id until + // its volume registers and a grow round finishes. + if err := f.waitForWritable(90 * time.Second); err != nil { + f.dumpLog("mount") + return fmt.Errorf("FUSE mount not writable: %v", err) + } + f.isSetup = true return nil } @@ -286,11 +294,16 @@ func (f *FuseTestFramework) dumpLog(name string) { f.t.Logf("[%s log tail (%d bytes)]\n%s", name, len(data), string(data)) } -// copyLogsForCI copies SeaweedFS process logs to /tmp/seaweedfs-fuse-logs/ -// so the CI workflow can upload them as artifacts. +// copyLogsForCI copies SeaweedFS process logs under a per-test directory in +// /tmp/seaweedfs-fuse-logs/ so the CI workflow can upload them as artifacts. +// Each framework instance gets its own directory: every test in the suite +// builds one, and fixed filenames would let a later passing test overwrite +// the logs of an earlier failing one. func (f *FuseTestFramework) copyLogsForCI() { - ciLogDir := "/tmp/seaweedfs-fuse-logs" - os.MkdirAll(ciLogDir, 0755) + ciLogDir := filepath.Join("/tmp/seaweedfs-fuse-logs", strings.NewReplacer("/", "_", " ", "_").Replace(f.t.Name())) + if err := os.MkdirAll(ciLogDir, 0755); err != nil { + return + } for _, name := range []string{"mini", "mount"} { src := filepath.Join(f.logDir, name+".log") data, err := os.ReadFile(src) @@ -421,6 +434,68 @@ func (f *FuseTestFramework) waitForMount(timeout time.Duration) error { return fmt.Errorf("mount point not ready within timeout") } +// waitForWritable probes the mount with a real write+read+remove until one +// succeeds, covering the window where a fresh cluster cannot yet assign a +// file id (volume not registered, grow pending, leader settle). Each attempt +// runs off the loop because a wedged mount can hold a file op open, and one +// attempt stays in flight until it finishes or the overall deadline passes. +func (f *FuseTestFramework) waitForWritable(timeout time.Duration) error { + probe := filepath.Join(f.mountPoint, ".framework_write_probe") + payload := []byte("probe") + deadline := time.Now().Add(timeout) + var lastErr error + for time.Now().Before(deadline) { + if exitErr := f.mountProcess.exited(); exitErr != nil { + return fmt.Errorf("mount process exited (%v)", exitErr) + } + result := make(chan error, 1) + go func() { + if err := os.WriteFile(probe, payload, 0644); err != nil { + result <- err + return + } + data, err := os.ReadFile(probe) + if err != nil { + result <- err + return + } + if len(data) != len(payload) { + result <- fmt.Errorf("read back %d bytes, want %d", len(data), len(payload)) + return + } + result <- os.Remove(probe) + }() + tick := time.NewTicker(500 * time.Millisecond) + attemptOpen := true + for attemptOpen { + select { + case err := <-result: + attemptOpen = false + if err == nil { + tick.Stop() + return nil + } + lastErr = err + case <-tick.C: + if exitErr := f.mountProcess.exited(); exitErr != nil { + tick.Stop() + return fmt.Errorf("mount process exited (%v)", exitErr) + } + if !time.Now().Before(deadline) { + tick.Stop() + if lastErr == nil { + return fmt.Errorf("write probe did not complete within timeout") + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) + } + } + } + tick.Stop() + time.Sleep(500 * time.Millisecond) + } + return fmt.Errorf("write probe failed within timeout: %v", lastErr) +} + // deviceID returns the device ID of the filesystem containing path. func deviceID(path string) (uint64, error) { var st syscall.Stat_t diff --git a/test/fuse_integration/posix_external_test.go b/test/fuse_integration/posix_external_test.go index 2c63fcb929..5a330fa2a4 100644 --- a/test/fuse_integration/posix_external_test.go +++ b/test/fuse_integration/posix_external_test.go @@ -102,10 +102,13 @@ func (s *ExternalPOSIXTestSuite) TestPjdFsTest(t *testing.T) { } } - // Run each test category + // pjdfstest asserts full POSIX semantics the mount does not provide + // yet, so category failures are report-only unless the run asked for + // external tools to gate. + strict := os.Getenv("POSIX_REQUIRE_EXTERNAL_TOOLS") == "true" for _, test := range pjdTests { t.Run(test.name, func(t *testing.T) { - s.runPjdTest(t, pjdDir, test.testPath, testDir, test.critical) + s.runPjdTest(t, pjdDir, test.testPath, testDir, strict && test.critical) }) } } From c925869dcc9f9b0c6e7307d90027a99833023ec4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 15:21:41 -0700 Subject: [PATCH 14/70] shell: retry filer discovery until a filer registers (#11994) * shell: retry filer discovery until a filer registers When -filer is not given, the shell discovers a filer with a single ListClusterNodes call right after connecting to the master. A filer can take several seconds to register after its port reports ready, so the list routinely comes back empty and the shell proceeds with an empty FilerAddress. Commands then fail with "dial tcp: missing address", which is how the s3tables Dremio integration test flaked in CI. Treat an empty list as retryable and keep polling ListClusterNodes until a filer appears or -connectTimeout expires. On timeout, print a clear "no filer registered" notice instead of failing later inside a command; master-only shell commands keep working either way. * shell: report the last filer discovery attempt failure A successful empty ListClusterNodes followed by a master outage left the diagnostic claiming no filer registered, hiding the real RPC failure. Track the outcome of the latest attempt instead: if the final list call failed, report that error; only when the last response was genuinely empty report that no filer registered within the timeout. * shell: test filer discovery retries against a fake master Extract the discovery loop into listFilers and cover the retry behavior with a fake master_pb.SeaweedServer: an empty cluster turning populated, an empty cluster exhausting the window without error, the last attempt's RPC failure being reported, and master-only commands still running after the discovery timeout. --- weed/shell/shell_liner.go | 60 ++++++-- weed/shell/shell_liner_test.go | 241 +++++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+), 15 deletions(-) diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 6aae477b0f..87d18c9b6c 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -3,6 +3,7 @@ package shell import ( "bufio" "context" + "errors" "fmt" "io" "math/rand/v2" @@ -25,6 +26,47 @@ import ( var historyPath = path.Join(os.TempDir(), "weed-shell") +// errNoFilersDiscovered marks a successful ListClusterNodes call that came +// back empty, which usually means the filer has not registered with the +// master yet. Unlike an RPC failure it is expected at startup, so filer +// discovery retries on it until the connect timeout expires. +var errNoFilersDiscovered = errors.New("no filer registered with master yet") + +// listFilers polls ListClusterNodes until at least one filer is registered +// with the master or ctx expires. An empty list is expected while a filer is +// still starting, so it retries like a transient failure. On return, a nil +// error with an empty list means the master answered but no filer registered +// in time; a non-nil error is the last attempt's failure. +func listFilers(ctx context.Context, commandEnv *CommandEnv, filerGroup string, maxWait time.Duration) (filers []pb.ServerAddress, listErr error) { + util.RetryWithBackoff(ctx, "filer discovery", maxWait, + func(err error) bool { + return errors.Is(err, errNoFilersDiscovered) || util.IsTransientError(err) + }, func() error { + filers = filers[:0] + listErr = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: filerGroup, + }) + if err != nil { + return err + } + for _, clusterNode := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(clusterNode.Address)) + } + return nil + }) + if listErr != nil { + return listErr + } + if len(filers) == 0 { + return errNoFilersDiscovered + } + return nil + }) + return filers, listErr +} + // Piped stdin returns the last command failure; an interactive session shows // the error to the operator and keeps going. func RunShell(options ShellOptions) error { @@ -75,27 +117,15 @@ func RunShell(options ShellOptions) error { } if commandEnv.option.FilerAddress == "" { - var filers []pb.ServerAddress discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout) defer cancelDiscover() - listErr := commandEnv.MasterClient.WithClient(discoverCtx, false, func(client master_pb.SeaweedClient) error { - resp, err := client.ListClusterNodes(discoverCtx, &master_pb.ListClusterNodesRequest{ - ClientType: cluster.FilerType, - FilerGroup: *options.FilerGroup, - }) - if err != nil { - return err - } - - for _, clusterNode := range resp.ClusterNodes { - filers = append(filers, pb.ServerAddress(clusterNode.Address)) - } - return nil - }) + filers, listErr := listFilers(discoverCtx, commandEnv, *options.FilerGroup, connectTimeout) if len(filers) > 0 { commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))] } else if listErr != nil { fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr) + } else { + fmt.Fprintf(os.Stderr, "no filer registered with master %s within %s\n", *options.Masters, connectTimeout) } if options.Debug { if len(filers) > 0 { diff --git a/weed/shell/shell_liner_test.go b/weed/shell/shell_liner_test.go index 21a707ac8a..d8268f887f 100644 --- a/weed/shell/shell_liner_test.go +++ b/weed/shell/shell_liner_test.go @@ -1,11 +1,22 @@ package shell import ( + "context" "errors" + "fmt" "io" + "net" + "os" "strings" + "sync/atomic" "testing" "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" ) type fakeCommand struct { @@ -83,3 +94,233 @@ func TestProcessEachCmdPreservesRegisteredCommandError(t *testing.T) { t.Errorf("a succeeding command must return no error, got exit=%v err=%v", exit, err) } } + +// fakeMaster implements just enough of master_pb.SeaweedServer for filer +// discovery: KeepConnected holds the stream open so the client marks the +// master connected, and ListClusterNodes answers from a script. +type fakeMaster struct { + master_pb.UnimplementedSeaweedServer + listCalls atomic.Int32 + onList func() (*master_pb.ListClusterNodesResponse, error) +} + +func (f *fakeMaster) KeepConnected(stream grpc.BidiStreamingServer[master_pb.KeepConnectedRequest, master_pb.KeepConnectedResponse]) error { + for { + if _, err := stream.Recv(); err != nil { + return err + } + } +} + +func (f *fakeMaster) ListClusterNodes(context.Context, *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) { + f.listCalls.Add(1) + return f.onList() +} + +func (f *fakeMaster) VolumeList(context.Context, *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) { + return &master_pb.VolumeListResponse{TopologyInfo: &master_pb.TopologyInfo{}}, nil +} + +// startFakeMaster serves the fake master on an ephemeral gRPC port and +// returns the http-form master address (whose gRPC port is port+10000) to +// hand to ShellOptions.Masters. +func startFakeMaster(t *testing.T) (*fakeMaster, string) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("fake master listen: %v", err) + } + grpcPort := listener.Addr().(*net.TCPAddr).Port + if grpcPort < 10001 { + listener.Close() + t.Fatalf("fake master needs a port above 10000 for its http form, got %d", grpcPort) + } + fake := &fakeMaster{} + server := grpc.NewServer() + master_pb.RegisterSeaweedServer(server, fake) + go server.Serve(listener) + t.Cleanup(server.Stop) + return fake, fmt.Sprintf("127.0.0.1:%d", grpcPort-10000) +} + +func newTestCommandEnv(t *testing.T, masters string) *CommandEnv { + t.Helper() + filerGroup := "" + options := &ShellOptions{ + Masters: &masters, + FilerGroup: &filerGroup, + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + } + commandEnv := NewCommandEnv(options) + ctx, cancel := context.WithCancel(context.Background()) + go commandEnv.MasterClient.KeepConnectedToMaster(ctx) + t.Cleanup(cancel) + return commandEnv +} + +func filerNode(address string) *master_pb.ListClusterNodesResponse { + return &master_pb.ListClusterNodesResponse{ + ClusterNodes: []*master_pb.ListClusterNodesResponse_ClusterNode{ + {Address: address}, + }, + } +} + +// A filer registering mid-discovery turns an empty list into a populated one +// on a later attempt. +func TestListFilersEmptyThenPopulated(t *testing.T) { + master, masters := startFakeMaster(t) + master.onList = func() (*master_pb.ListClusterNodesResponse, error) { + if master.listCalls.Load() <= 2 { + return &master_pb.ListClusterNodesResponse{}, nil + } + return filerNode("127.0.0.1:9888"), nil + } + + commandEnv := newTestCommandEnv(t, masters) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + filers, err := listFilers(ctx, commandEnv, "", 10*time.Second) + if err != nil { + t.Fatalf("listFilers failed: %v", err) + } + if len(filers) != 1 || filers[0] != "127.0.0.1:9888" { + t.Fatalf("expected the late-registered filer, got %v", filers) + } + if got := master.listCalls.Load(); got != 3 { + t.Fatalf("expected 3 discovery attempts, got %d", got) + } +} + +// A master that answers but never lists a filer exhausts the discovery +// window and reports an empty, error-free result — the caller then prints +// the "no filer registered" notice and continues without a filer. +func TestListFilersEmptyUntilTimeout(t *testing.T) { + master, masters := startFakeMaster(t) + master.onList = func() (*master_pb.ListClusterNodesResponse, error) { + return &master_pb.ListClusterNodesResponse{}, nil + } + commandEnv := newTestCommandEnv(t, masters) + + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + + filers, err := listFilers(ctx, commandEnv, "", 1500*time.Millisecond) + if err != nil { + t.Fatalf("empty discovery must not return an error, got: %v", err) + } + if len(filers) != 0 { + t.Fatalf("expected no filers, got %v", filers) + } + if got := master.listCalls.Load(); got < 2 { + t.Fatalf("expected repeated polling, got %d attempts", got) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Fatalf("discovery should give up near the timeout, took %v", elapsed) + } +} + +// A non-transient failure on the last attempt surfaces as the discovery +// error, so the diagnostic reports the RPC failure instead of a missing +// filer. +func TestListFilersReportsLastError(t *testing.T) { + master, masters := startFakeMaster(t) + master.onList = func() (*master_pb.ListClusterNodesResponse, error) { + return nil, status.Error(codes.PermissionDenied, "denied") + } + commandEnv := newTestCommandEnv(t, masters) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + filers, err := listFilers(ctx, commandEnv, "", 10*time.Second) + if err == nil { + t.Fatalf("expected the permission error to be reported") + } + if len(filers) != 0 { + t.Fatalf("expected no filers, got %v", filers) + } + if got := master.listCalls.Load(); got != 1 { + t.Fatalf("a non-transient error must stop retries, got %d attempts", got) + } +} + +// A transient RPC failure on the last attempt also surfaces as the discovery +// error rather than as a missing filer. +func TestListFilersErrorAfterEmptyList(t *testing.T) { + master, masters := startFakeMaster(t) + master.onList = func() (*master_pb.ListClusterNodesResponse, error) { + if master.listCalls.Load() <= 1 { + return &master_pb.ListClusterNodesResponse{}, nil + } + return nil, status.Error(codes.Unavailable, "master gone") + } + commandEnv := newTestCommandEnv(t, masters) + + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + + filers, err := listFilers(ctx, commandEnv, "", 1500*time.Millisecond) + if err == nil { + t.Fatalf("expected the late RPC failure to be reported") + } + if len(filers) != 0 { + t.Fatalf("expected no filers, got %v", filers) + } +} + +// After discovery times out with no filer, master-only commands still run: +// the shell warns about the missing filer but proceeds to execute stdin. +func TestRunShellFilerDiscoveryTimeout(t *testing.T) { + master, masters := startFakeMaster(t) + master.onList = func() (*master_pb.ListClusterNodesResponse, error) { + return &master_pb.ListClusterNodesResponse{}, nil + } + filerGroup := "" + + oldStdin := os.Stdin + inReader, inWriter, err := os.Pipe() + if err != nil { + t.Fatalf("stdin pipe: %v", err) + } + os.Stdin = inReader + t.Cleanup(func() { os.Stdin = oldStdin }) + go func() { + fmt.Fprintln(inWriter, "cluster.ps") + fmt.Fprintln(inWriter, "exit") + inWriter.Close() + }() + + oldStderr := os.Stderr + errReader, errWriter, err := os.Pipe() + if err != nil { + t.Fatalf("stderr pipe: %v", err) + } + os.Stderr = errWriter + t.Cleanup(func() { os.Stderr = oldStderr }) + + done := make(chan error, 1) + go func() { + done <- RunShell(ShellOptions{ + Masters: &masters, + FilerGroup: &filerGroup, + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + ConnectTimeout: 1500 * time.Millisecond, + }) + }() + + select { + case err = <-done: + case <-time.After(15 * time.Second): + t.Fatalf("RunShell did not return after discovery timeout") + } + errWriter.Close() + stderr, _ := io.ReadAll(errReader) + if err != nil { + t.Fatalf("master-only commands must run without a filer, got: %v", err) + } + if !strings.Contains(string(stderr), "no filer registered") { + t.Fatalf("expected the missing-filer notice on stderr, got: %q", stderr) + } +} From aeb5db1b13055f24b443c684ee77eff0779cb4c1 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 15:38:09 -0700 Subject: [PATCH 15/70] tests: pin weed mini admin ports and disable unused aux services (#11997) * tests: pin weed mini admin ports and disable unused aux services Every weed mini that does not configure them still binds the fixed default ports for the admin server (12535 + derived 22535 gRPC), its legacy compat listener (23646 + 33646), WebDAV (7333), Lance (9101) and the Iceberg REST catalog (8181). Sequential or concurrent minis on one host then collide on them: the port check warns and shifts, and the admin's late bind can land on a stolen ephemeral port, adding noise and flakiness to test logs (e.g. run 40017181958, job 120659267479). Add testutil.MiniAdminArgs to pin the admin server to an allocated mini-style port (HTTP + port+10000 gRPC, both below the Linux ephemeral port floor since admin binds late) and disable the legacy compat listener. Every test that spawns weed mini now also disables the aux services it does not use: WebDAV, Lance and the Iceberg REST endpoint get port=0 unless the test actually exercises them, in which case their ports stay pinned as before. * tests: take mini admin ports from the below-ephemeral range * tests: reuse MiniAdminArgs for the remote-s3 fetch mini * tests: fully disable webdav and co-allocate mini admin ports - -webdav.port=0 only makes the service bind a random port; add -webdav=false so minis that do not test WebDAV skip it entirely. - Spark, RisingWave and s3 Spark launchers allocated the admin port in a second MustFreeMiniPort call, which could pick a port already handed out by the first batch. Fold 'Admin' into the same MustFreeMiniPorts call. - test/fuse_integration is a separate Go module and cannot import testutil; allocate the admin ports with its local freePort helper instead. * test/fuse: derive the mini admin gRPC port from its HTTP port Two independent freePort calls could return the same port, making admin.port equal admin.port.grpc (or collide with the filer port). Allocate once and derive the gRPC port with the +10000 convention. * test/fuse: pin the mini admin port in the fuse_test framework copy The fuse_test package framework spawns weed mini too; without a pinned admin port it still binds the fixed defaults 12535/23646 and collides with other minis. * tests: drop the redundant -webdav.port=0 flag -webdav=false alone disables the service; the port flag is dead config. --- test/fuse_integration/framework.go | 7 +++++++ test/fuse_integration/framework_test.go | 7 +++++++ test/s3/iam/s3_iam_admin_test.go | 4 ++-- .../iam/s3_sts_get_federation_token_test.go | 2 +- .../s3_lifecycle_object_lock_test.go | 1 - test/s3/normal/s3_integration_test.go | 7 +++++-- test/s3/policy/policy_test.go | 11 +++++----- .../remote_cache/remote_cache_evict_test.go | 8 ++++++-- .../sdk_v2_routing/s3_sdk_v2_routing_test.go | 1 - test/s3/spark/setup_test.go | 9 +++++---- .../admin/admin_iceberg_pages_test.go | 16 ++++++++++----- test/s3tables/catalog/duckdb_oauth_test.go | 2 ++ test/s3tables/catalog/iceberg_catalog_test.go | 12 +++++++++-- .../clickhouse_catalog_test.go | 6 ++++-- .../catalog_doris/doris_catalog_test.go | 6 ++++-- .../catalog_dremio/dremio_catalog_test.go | 6 ++++-- .../catalog_duckdb_lance/duckdb_lance_test.go | 6 ++++-- .../catalog_lancedb/lancedb_catalog_test.go | 6 ++++-- .../catalog_olake/olake_catalog_test.go | 6 ++++-- .../s3tables/catalog_risingwave/setup_test.go | 9 +++++---- test/s3tables/catalog_spark/setup_test.go | 9 +++++---- .../catalog_spark_lance/spark_lance_test.go | 6 ++++-- .../catalog_trino/trino_catalog_test.go | 6 ++++-- test/s3tables/lakekeeper/lakekeeper_test.go | 6 ++++-- test/s3tables/lifecycle/cluster_test.go | 12 +++++++++-- .../maintenance_integration_test.go | 16 ++++++++++----- test/s3tables/polaris/polaris_env_test.go | 6 ++++-- .../sts_integration/sts_integration_test.go | 6 ++++-- .../s3tables_integration_test.go | 20 ++++++++++++------- test/s3tables/unity_catalog/setup_test.go | 2 ++ test/testutil/ports.go | 16 +++++++++++++++ .../grpc/fetch_remote_s3_test.go | 11 +++++++--- 32 files changed, 176 insertions(+), 72 deletions(-) diff --git a/test/fuse_integration/framework.go b/test/fuse_integration/framework.go index a6d002eeef..9fe62fcd71 100644 --- a/test/fuse_integration/framework.go +++ b/test/fuse_integration/framework.go @@ -424,6 +424,10 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { // -v is a global flag; before the subcommand or flag parsing dies args = append(args, "-v=4") } + adminPort := freePort(f.t) + for adminPort == f.filerPort { + adminPort = freePort(f.t) + } args = append(args, "mini", "-dir="+f.dataDir, @@ -433,6 +437,9 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { "-s3=false", "-webdav=false", "-admin.ui=false", + "-admin.port="+strconv.Itoa(adminPort), + "-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset), + "-admin.port.legacy=0", ) proc, err := f.startProcess("mini", args) diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index f045fa157b..3141d9c51d 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -320,6 +320,10 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { if config.EnableDebug { args = append(args, "-v=4") } + adminPort := freePort(f.t) + for adminPort == f.filerPort { + adminPort = freePort(f.t) + } args = append(args, "mini", "-dir="+f.dataDir, @@ -329,6 +333,9 @@ func (f *FuseTestFramework) startMini(config *TestConfig) error { "-s3=false", "-webdav=false", "-admin.ui=false", + "-admin.port="+strconv.Itoa(adminPort), + "-admin.port.grpc="+strconv.Itoa(adminPort+grpcPortOffset), + "-admin.port.legacy=0", ) proc, err := f.startProcess("mini", args) diff --git a/test/s3/iam/s3_iam_admin_test.go b/test/s3/iam/s3_iam_admin_test.go index 75556b43f7..70906dd26e 100644 --- a/test/s3/iam/s3_iam_admin_test.go +++ b/test/s3/iam/s3_iam_admin_test.go @@ -295,8 +295,8 @@ func TestIAMPolicyManagement(t *testing.T) { assert.Equal(t, true, aws.BoolValue(listVersionsResp.Versions[0].IsDefaultVersion)) getVersionResp, err := iamClient.GetPolicyVersion(&iam.GetPolicyVersionInput{ - PolicyArn: createResp.Policy.Arn, - VersionId: aws.String("v1"), + PolicyArn: createResp.Policy.Arn, + VersionId: aws.String("v1"), }) require.NoError(t, err) require.NotNil(t, getVersionResp.PolicyVersion) diff --git a/test/s3/iam/s3_sts_get_federation_token_test.go b/test/s3/iam/s3_sts_get_federation_token_test.go index 2a718cba90..69807bb169 100644 --- a/test/s3/iam/s3_sts_get_federation_token_test.go +++ b/test/s3/iam/s3_sts_get_federation_token_test.go @@ -14,8 +14,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" v4 "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/aws/aws-sdk-go/service/s3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/test/s3/lifecycle/s3_lifecycle_object_lock_test.go b/test/s3/lifecycle/s3_lifecycle_object_lock_test.go index 7196346de9..e601f85500 100644 --- a/test/s3/lifecycle/s3_lifecycle_object_lock_test.go +++ b/test/s3/lifecycle/s3_lifecycle_object_lock_test.go @@ -116,4 +116,3 @@ func TestLifecycleSkipsObjectLockedObjects(t *testing.T) { }) require.NoError(t, err, "object under GOVERNANCE retention must survive the sweep") } - diff --git a/test/s3/normal/s3_integration_test.go b/test/s3/normal/s3_integration_test.go index e9a9ed6908..e37972c4bb 100644 --- a/test/s3/normal/s3_integration_test.go +++ b/test/s3/normal/s3_integration_test.go @@ -117,7 +117,8 @@ func TestS3Integration(t *testing.T) { // Extra flags can be appended via extraArgs. func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) { // Allocate non-colliding ports (including gRPC offsets) for weed mini. - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3", "Admin"}) + adminPort := ports[4] masterPort := ports[0] volumePort := ports[1] filerPort := ports[2] @@ -186,13 +187,15 @@ func startMiniCluster(t *testing.T, extraArgs ...string) (*TestCluster, error) { "-volume.port=" + strconv.Itoa(volumePort), "-filer.port=" + strconv.Itoa(filerPort), "-s3.port=" + strconv.Itoa(s3Port), - "-webdav.port=0", // Disable WebDAV + "-webdav=false", // Disable WebDAV "-admin.ui=false", // Disable admin UI "-master.volumeSizeLimitMB=32", // Small volumes for testing "-ip=127.0.0.1", "-master.peers=none", // Faster startup "-s3.iam.readOnly=false", // Enable IAM write operations for tests }, extraArgs...) + os.Args = append(os.Args, testutil.MiniAdminArgs(adminPort)...) + os.Args = append(os.Args, "-s3.port.lance=0", "-s3.port.iceberg=0") // Suppress most logging during tests glog.MaxSize = 1024 * 1024 diff --git a/test/s3/policy/policy_test.go b/test/s3/policy/policy_test.go index 2e3e1364e3..302ddbceb5 100644 --- a/test/s3/policy/policy_test.go +++ b/test/s3/policy/policy_test.go @@ -710,12 +710,12 @@ func startMiniCluster(t *testing.T) (*TestCluster, error) { // mini in a goroutine, but the previous mini's admin goroutine is // still binding 12535 — so the next test's admin can never become // ready and mini.go fatals on its readiness check. - ports := testutil.MustAllocatePorts(t, 10) + ports := testutil.MustAllocatePorts(t, 8) + adminPort := testutil.MustFreeMiniPort(t, "admin") masterPort, masterGrpcPort := ports[0], ports[1] volumePort, volumeGrpcPort := ports[2], ports[3] filerPort, filerGrpcPort := ports[4], ports[5] s3Port, s3GrpcPort := ports[6], ports[7] - adminPort, adminGrpcPort := ports[8], ports[9] // Manually-managed temp dir (not t.TempDir()) so we control removal order: // the dir is removed inside Stop() AFTER the mini goroutine has fully @@ -786,15 +786,16 @@ enabled = true "-filer.port.grpc=" + strconv.Itoa(filerGrpcPort), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3GrpcPort), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpcPort), - "-webdav.port=0", + "-webdav=false", + "-s3.port.lance=0", + "-s3.port.iceberg=0", "-admin.ui=false", "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + os.Args = append(os.Args, testutil.MiniAdminArgs(adminPort)...) glog.MaxSize = 1024 * 1024 for _, cmd := range command.Commands { if cmd.Name() == "mini" && cmd.Run != nil { diff --git a/test/s3/remote_cache/remote_cache_evict_test.go b/test/s3/remote_cache/remote_cache_evict_test.go index 49060319f4..cb44f1f3fb 100644 --- a/test/s3/remote_cache/remote_cache_evict_test.go +++ b/test/s3/remote_cache/remote_cache_evict_test.go @@ -16,6 +16,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/test/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -48,12 +49,15 @@ func startMini(t *testing.T, dir string, args ...string) { logFile, err := os.Create(logPath) require.NoError(t, err) miniLogPaths = append(miniLogPaths, logPath) - cmd := exec.Command(weedBinary, append([]string{"mini", + miniArgs := append([]string{"mini", "-dir=" + dir, "-s3.config=s3_config.json", "-s3.allowDeleteBucketNotEmpty=true", "-ip=127.0.0.1", "-ip.bind=127.0.0.1", - }, args...)...) + }, args...) + miniArgs = append(miniArgs, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.Command(weedBinary, miniArgs...) cmd.Stdout = logFile cmd.Stderr = logFile require.NoError(t, cmd.Start()) diff --git a/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go b/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go index 2345fdf573..9adc2300f2 100644 --- a/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go +++ b/test/s3/sdk_v2_routing/s3_sdk_v2_routing_test.go @@ -207,4 +207,3 @@ func TestCreateAndListBuckets_ServiceLevel(t *testing.T) { } require.True(t, found, "bucket %q must appear in service-level ListBuckets", bucket) } - diff --git a/test/s3/spark/setup_test.go b/test/s3/spark/setup_test.go index efdb2c60d5..3b647a2f8e 100644 --- a/test/s3/spark/setup_test.go +++ b/test/s3/spark/setup_test.go @@ -94,7 +94,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -105,8 +105,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create IAM config: %v", err) } - env.masterProcess = exec.Command( - env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-ip", bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -114,7 +113,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port", fmt.Sprintf("%d", env.s3Port), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[3])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + env.masterProcess = exec.Command(env.weedBinary, miniArgs...) weedLogPath := filepath.Join(env.seaweedfsDataDir, "weed-mini.log") weedLogFile, err := os.Create(weedLogPath) if err != nil { diff --git a/test/s3tables/admin/admin_iceberg_pages_test.go b/test/s3tables/admin/admin_iceberg_pages_test.go index 4b6b2e4102..1cc32c2168 100644 --- a/test/s3tables/admin/admin_iceberg_pages_test.go +++ b/test/s3tables/admin/admin_iceberg_pages_test.go @@ -91,7 +91,11 @@ func TestMain(m *testing.M) { } func startCluster(testDir string) (*testCluster, error) { - ports, err := testutil.AllocatePorts(10) + ports, err := testutil.AllocatePorts(9) + if err != nil { + return nil, err + } + adminPorts, err := testutil.AllocateMiniPorts(1) if err != nil { return nil, err } @@ -99,7 +103,8 @@ func startCluster(testDir string) (*testCluster, error) { volumePort, volumeGrpc := ports[2], ports[3] filerPort, filerGrpc := ports[4], ports[5] s3Port, s3Grpc := ports[6], ports[7] - adminPort, adminGrpc := ports[8], ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] // Empty security.toml disables JWT auth. if err := os.WriteFile(filepath.Join(testDir, "security.toml"), []byte("# test\n"), 0644); err != nil { @@ -150,14 +155,15 @@ func startCluster(testDir string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpc), - "-webdav.port=0", + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + args = append(args, testutil.MiniAdminArgs(adminPort)...) os.Args = append([]string{"weed"}, args...) glog.MaxSize = 1024 * 1024 for _, cmd := range command.Commands { diff --git a/test/s3tables/catalog/duckdb_oauth_test.go b/test/s3tables/catalog/duckdb_oauth_test.go index 865a7ee8d5..346cd2d096 100644 --- a/test/s3tables/catalog/duckdb_oauth_test.go +++ b/test/s3tables/catalog/duckdb_oauth_test.go @@ -135,6 +135,8 @@ func (env *oauthTestEnv) start(t *testing.T) { if env.s3ExternalURL != "" { args = append(args, "-s3.externalUrl", env.s3ExternalURL) } + args = append(args, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + args = append(args, "-s3.port.lance=0") cmd := exec.CommandContext(ctx, env.weedBinary, args...) cmd.Dir = env.dataDir diff --git a/test/s3tables/catalog/iceberg_catalog_test.go b/test/s3tables/catalog/iceberg_catalog_test.go index b45f3714ad..908f54b176 100644 --- a/test/s3tables/catalog/iceberg_catalog_test.go +++ b/test/s3tables/catalog/iceberg_catalog_test.go @@ -63,6 +63,7 @@ type TestEnvironment struct { s3GrpcPort int icebergPort int lancePort int + adminPort int masterPort int masterGrpcPort int filerPort int @@ -117,6 +118,10 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) { if err != nil { return nil, fmt.Errorf("allocate ports: %w", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("allocate admin port: %w", err) + } return &TestEnvironment{ seaweedDir: seaweedDir, @@ -132,6 +137,7 @@ func newTestEnvironmentForMain() (*TestEnvironment, error) { volumePort: ports[7], volumeGrpcPort: ports[8], lancePort: ports[9], + adminPort: adminPorts[0], dockerAvailable: testutil.HasDocker(), }, nil } @@ -152,7 +158,7 @@ func (env *TestEnvironment) startSeaweedFSForMain() error { } } - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -165,7 +171,9 @@ func (env *TestEnvironment) startSeaweedFSForMain() error { "-s3.port.lance", fmt.Sprintf("%d", env.lancePort), "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(env.adminPort)...) + miniArgs = append(miniArgs, "-webdav=false") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go index d12173398c..098fae43a5 100644 --- a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go +++ b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go @@ -364,7 +364,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -378,7 +378,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_doris/doris_catalog_test.go b/test/s3tables/catalog_doris/doris_catalog_test.go index b7b071b3f2..3a8aaf9e22 100644 --- a/test/s3tables/catalog_doris/doris_catalog_test.go +++ b/test/s3tables/catalog_doris/doris_catalog_test.go @@ -303,7 +303,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -317,7 +317,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_dremio/dremio_catalog_test.go b/test/s3tables/catalog_dremio/dremio_catalog_test.go index 09eed0055e..35fedb00f1 100644 --- a/test/s3tables/catalog_dremio/dremio_catalog_test.go +++ b/test/s3tables/catalog_dremio/dremio_catalog_test.go @@ -289,7 +289,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -303,7 +303,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go b/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go index 997cab2601..5cefd12059 100644 --- a/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go +++ b/test/s3tables/catalog_duckdb_lance/duckdb_lance_test.go @@ -142,7 +142,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -156,7 +156,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go index 85a31861f6..96613937c0 100644 --- a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go +++ b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go @@ -137,7 +137,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -151,7 +151,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_olake/olake_catalog_test.go b/test/s3tables/catalog_olake/olake_catalog_test.go index eafd815e11..bbed0c2a3d 100644 --- a/test/s3tables/catalog_olake/olake_catalog_test.go +++ b/test/s3tables/catalog_olake/olake_catalog_test.go @@ -275,7 +275,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("Failed to create IAM config: %v", err) } - cmd := exec.Command(env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -292,7 +292,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.Command(env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_risingwave/setup_test.go b/test/s3tables/catalog_risingwave/setup_test.go index b893c1d91a..155c390eed 100644 --- a/test/s3tables/catalog_risingwave/setup_test.go +++ b/test/s3tables/catalog_risingwave/setup_test.go @@ -113,7 +113,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -134,8 +134,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { env.logFile = logFile // Start SeaweedFS using weed mini (all-in-one including Iceberg REST) - env.masterProcess = exec.Command( - "weed", "mini", + miniArgs := append([]string{"mini", "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -144,7 +143,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[4])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + env.masterProcess = exec.Command("weed", miniArgs...) env.masterProcess.Stdout = logFile env.masterProcess.Stderr = logFile env.masterProcess.Env = append(os.Environ(), diff --git a/test/s3tables/catalog_spark/setup_test.go b/test/s3tables/catalog_spark/setup_test.go index 9091b201c8..7d64dfd6eb 100644 --- a/test/s3tables/catalog_spark/setup_test.go +++ b/test/s3tables/catalog_spark/setup_test.go @@ -90,7 +90,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { t.Fatalf("failed to create temp directory: %v", err) } - ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg"}) + ports := testutil.MustFreeMiniPorts(t, []string{"Master", "Filer", "S3", "Iceberg", "Admin"}) env.masterPort = ports[0] env.filerPort = ports[1] env.s3Port = ports[2] @@ -104,8 +104,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { } // Start SeaweedFS using weed mini (all-in-one including Iceberg REST) - env.masterProcess = exec.Command( - "weed", "mini", + miniArgs := append([]string{"mini", "-ip", bindIP, "-ip.bind", "0.0.0.0", "-master.port", fmt.Sprintf("%d", env.masterPort), @@ -114,7 +113,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-s3.port.iceberg", fmt.Sprintf("%d", env.icebergRestPort), "-s3.config", iamConfigPath, "-dir", env.seaweedfsDataDir, - ) + }, testutil.MiniAdminArgs(ports[4])...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + env.masterProcess = exec.Command("weed", miniArgs...) env.masterProcess.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID="+env.accessKey, "AWS_SECRET_ACCESS_KEY="+env.secretKey, diff --git a/test/s3tables/catalog_spark_lance/spark_lance_test.go b/test/s3tables/catalog_spark_lance/spark_lance_test.go index d5d808096d..5bf1a0359f 100644 --- a/test/s3tables/catalog_spark_lance/spark_lance_test.go +++ b/test/s3tables/catalog_spark_lance/spark_lance_test.go @@ -137,7 +137,7 @@ func (env *environment) start(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -151,7 +151,9 @@ func (env *environment) start(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/catalog_trino/trino_catalog_test.go b/test/s3tables/catalog_trino/trino_catalog_test.go index 7645dd2a8a..ddfc60fdeb 100644 --- a/test/s3tables/catalog_trino/trino_catalog_test.go +++ b/test/s3tables/catalog_trino/trino_catalog_test.go @@ -256,7 +256,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -270,7 +270,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/lakekeeper/lakekeeper_test.go b/test/s3tables/lakekeeper/lakekeeper_test.go index 7076cf187a..d3a23c76c4 100644 --- a/test/s3tables/lakekeeper/lakekeeper_test.go +++ b/test/s3tables/lakekeeper/lakekeeper_test.go @@ -205,7 +205,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { env.weedCancel = cancel // Start weed mini with both S3 config (standard IAM) and IAM config (advanced IAM/STS) - cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini", + miniArgs := append([]string{"-v", "4", "mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -220,7 +220,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/lifecycle/cluster_test.go b/test/s3tables/lifecycle/cluster_test.go index 2ab04812e3..ce5037f6b5 100644 --- a/test/s3tables/lifecycle/cluster_test.go +++ b/test/s3tables/lifecycle/cluster_test.go @@ -99,6 +99,7 @@ type environment struct { s3GrpcPort int icebergPort int lancePort int + adminPort int weedCancel context.CancelFunc weedCmd *exec.Cmd @@ -138,6 +139,10 @@ func newEnvironment() (*environment, error) { if err != nil { return nil, fmt.Errorf("allocate ports: %w", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("allocate admin port: %w", err) + } return &environment{ weedBinary: weedBinary, rootDir: seaweedDir, @@ -153,6 +158,7 @@ func newEnvironment() (*environment, error) { s3GrpcPort: ports[7], icebergPort: ports[8], lancePort: ports[9], + adminPort: adminPorts[0], }, nil } @@ -166,7 +172,7 @@ func (env *environment) start() error { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -180,7 +186,9 @@ func (env *environment) start() error { "-s3.config", iamConfigPath, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(env.adminPort)...) + miniArgs = append(miniArgs, "-webdav=false") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/maintenance/maintenance_integration_test.go b/test/s3tables/maintenance/maintenance_integration_test.go index 4a5b763497..a92ef92657 100644 --- a/test/s3tables/maintenance/maintenance_integration_test.go +++ b/test/s3tables/maintenance/maintenance_integration_test.go @@ -92,7 +92,11 @@ func TestMain(m *testing.M) { } func startCluster(testDir string, extraArgs []string) (*testCluster, error) { - ports, err := testutil.AllocatePorts(10) + ports, err := testutil.AllocatePorts(9) + if err != nil { + return nil, err + } + adminPorts, err := testutil.AllocateMiniPorts(1) if err != nil { return nil, err } @@ -100,7 +104,8 @@ func startCluster(testDir string, extraArgs []string) (*testCluster, error) { volumePort, volumeGrpc := ports[2], ports[3] filerPort, filerGrpc := ports[4], ports[5] s3Port, s3Grpc := ports[6], ports[7] - adminPort, adminGrpc := ports[8], ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] _ = os.Remove(filepath.Join(testDir, "mini.options")) @@ -146,15 +151,16 @@ func startCluster(testDir string, extraArgs []string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpc), - "-webdav.port=0", + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", "-master.volumeSizeLimitMB=32", "-ip=127.0.0.1", "-master.peers=none", "-s3.iam.readOnly=false", } + args = append(args, testutil.MiniAdminArgs(adminPort)...) args = append(args, extraArgs...) os.Args = append([]string{"weed"}, args...) glog.MaxSize = 1024 * 1024 diff --git a/test/s3tables/polaris/polaris_env_test.go b/test/s3tables/polaris/polaris_env_test.go index fe286e18ab..8a0c58a505 100644 --- a/test/s3tables/polaris/polaris_env_test.go +++ b/test/s3tables/polaris/polaris_env_test.go @@ -182,7 +182,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "-v", "4", "mini", + miniArgs := append([]string{"-v", "4", "mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -197,7 +197,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/sts_integration/sts_integration_test.go b/test/s3tables/sts_integration/sts_integration_test.go index 672043cab3..524713fe47 100644 --- a/test/s3tables/sts_integration/sts_integration_test.go +++ b/test/s3tables/sts_integration/sts_integration_test.go @@ -242,7 +242,7 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) env.weedCancel = cancel - cmd := exec.CommandContext(ctx, env.weedBinary, "mini", + miniArgs := append([]string{"mini", "-master.port", fmt.Sprintf("%d", env.masterPort), "-master.port.grpc", fmt.Sprintf("%d", env.masterGrpcPort), "-volume.port", fmt.Sprintf("%d", env.volumePort), @@ -257,7 +257,9 @@ func (env *TestEnvironment) StartSeaweedFS(t *testing.T) { "-ip", env.bindIP, "-ip.bind", "0.0.0.0", "-dir", env.dataDir, - ) + }, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + miniArgs = append(miniArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") + cmd := exec.CommandContext(ctx, env.weedBinary, miniArgs...) cmd.Dir = env.dataDir cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr diff --git a/test/s3tables/table-buckets/s3tables_integration_test.go b/test/s3tables/table-buckets/s3tables_integration_test.go index 80b711e478..c99d6d4393 100644 --- a/test/s3tables/table-buckets/s3tables_integration_test.go +++ b/test/s3tables/table-buckets/s3tables_integration_test.go @@ -594,11 +594,16 @@ func testTargetOperations(t *testing.T, client *S3TablesClient) { // directory. It does not require a *testing.T so it can be called from TestMain. // extraArgs are appended to the default mini command flags. func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, error) { - // We need 10 unique ports: Master(2), Volume(2), Filer(2), S3(2), Admin(2) - ports, err := testutil.AllocatePorts(10) + // We need 9 unique ports: Master(2), Volume(2), Filer(2), S3(2), Iceberg(1); + // the admin pair comes from the mini-safe range. + ports, err := testutil.AllocatePorts(9) if err != nil { return nil, fmt.Errorf("failed to find available ports: %v", err) } + adminPorts, err := testutil.AllocateMiniPorts(1) + if err != nil { + return nil, fmt.Errorf("failed to allocate admin port: %v", err) + } masterPort := ports[0] masterGrpcPort := ports[1] @@ -608,8 +613,8 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er filerGrpcPort := ports[5] s3Port := ports[6] s3GrpcPort := ports[7] - adminPort := ports[8] - adminGrpcPort := ports[9] + adminPort := adminPorts[0] + icebergPort := ports[8] // Ensure no configuration file from previous runs configFile := filepath.Join(testDir, "mini.options") @@ -674,15 +679,16 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er "-filer.port.grpc=" + strconv.Itoa(filerGrpcPort), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3GrpcPort), - "-admin.port=" + strconv.Itoa(adminPort), - "-admin.port.grpc=" + strconv.Itoa(adminGrpcPort), - "-webdav.port=0", // Disable WebDAV + "-webdav=false", // Disable WebDAV + "-s3.port.lance=0", // Disable Lance + "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", // Disable admin UI "-master.volumeSizeLimitMB=32", // Small volumes for testing "-ip=127.0.0.1", "-master.peers=none", // Faster startup "-s3.iam.readOnly=false", // Enable IAM write operations for tests } + baseArgs = append(baseArgs, testutil.MiniAdminArgs(adminPort)...) if len(extraArgs) > 0 { baseArgs = append(baseArgs, extraArgs...) } diff --git a/test/s3tables/unity_catalog/setup_test.go b/test/s3tables/unity_catalog/setup_test.go index b56044cb40..3309aff254 100644 --- a/test/s3tables/unity_catalog/setup_test.go +++ b/test/s3tables/unity_catalog/setup_test.go @@ -183,6 +183,8 @@ func (env *testEnv) startSeaweedFS(t *testing.T, iamJSON string) { if enableSTS { weedArgs = append(weedArgs, "-s3.iam.config", iamConfigPath, "-s3.iam.readOnly=false") } + weedArgs = append(weedArgs, testutil.MiniAdminArgs(testutil.MustFreeMiniPort(t, "admin"))...) + weedArgs = append(weedArgs, "-webdav=false", "-s3.port.lance=0", "-s3.port.iceberg=0") cmd := exec.CommandContext(ctx, env.weedBinary, weedArgs...) cmd.Dir = env.dataDir diff --git a/test/testutil/ports.go b/test/testutil/ports.go index 0950cf5209..5a1ce76d46 100644 --- a/test/testutil/ports.go +++ b/test/testutil/ports.go @@ -167,6 +167,22 @@ func MustFreeMiniPort(t *testing.T, name string) int { return MustFreeMiniPorts(t, []string{name})[0] } +// MiniAdminArgs returns weed mini flags pinning the admin server to +// adminPort and adminPort+GrpcPortOffset and disabling the legacy compat +// listener. Unconfigured minis all bind the fixed default admin port 12535 +// and the legacy port 23646 (+33646 gRPC), so sequential or concurrent +// minis on one host collide on them. Allocate adminPort with +// MustFreeMiniPort(s) — the admin binds late in mini startup, after the +// cluster's own outbound connections may steal an ephemeral port, so it +// must stay below the Linux ephemeral port floor. +func MiniAdminArgs(adminPort int) []string { + return []string{ + fmt.Sprintf("-admin.port=%d", adminPort), + fmt.Sprintf("-admin.port.grpc=%d", adminPort+GrpcPortOffset), + "-admin.port.legacy=0", + } +} + // AllocatePortSet atomically allocates miniCount master-style port pairs (each // with port and port+GrpcPortOffset reserved) plus regularCount additional // distinct ports. All listeners are held open until the entire batch is diff --git a/test/volume_server/grpc/fetch_remote_s3_test.go b/test/volume_server/grpc/fetch_remote_s3_test.go index d3c9aa673e..856b712357 100644 --- a/test/volume_server/grpc/fetch_remote_s3_test.go +++ b/test/volume_server/grpc/fetch_remote_s3_test.go @@ -37,23 +37,28 @@ func startWeedMini(t *testing.T) (s3Endpoint string, cleanup func()) { } } - miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3"}) + miniPorts := testutil.MustFreeMiniPorts(t, []string{"Master", "Volume", "Filer", "S3", "Admin"}) miniMasterPort := miniPorts[0] miniVolumePort := miniPorts[1] miniFilerPort := miniPorts[2] miniS3Port := miniPorts[3] + miniAdminPort := miniPorts[4] miniDir := t.TempDir() os.WriteFile(filepath.Join(miniDir, "security.toml"), []byte("# empty\n"), 0644) ctx, cancel := context.WithCancel(context.Background()) - miniCmd := exec.CommandContext(ctx, weedBin, "mini", + miniArgs := append([]string{"mini", fmt.Sprintf("-dir=%s", miniDir), fmt.Sprintf("-master.port=%d", miniMasterPort), fmt.Sprintf("-volume.port=%d", miniVolumePort), fmt.Sprintf("-filer.port=%d", miniFilerPort), fmt.Sprintf("-s3.port=%d", miniS3Port), - ) + "-webdav=false", + "-s3.port.lance=0", + "-s3.port.iceberg=0", + }, testutil.MiniAdminArgs(miniAdminPort)...) + miniCmd := exec.CommandContext(ctx, weedBin, miniArgs...) miniCmd.Env = append(os.Environ(), "AWS_ACCESS_KEY_ID=admin", "AWS_SECRET_ACCESS_KEY=admin") miniCmd.Dir = miniDir logFile, _ := os.CreateTemp("", "weed-mini-*.log") From 29b81a827ded2bee34f6cbe8439d5509a63325cc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 15:44:18 -0700 Subject: [PATCH 16/70] volume: also listen on localhost like other components (#11995) * volume: also listen on localhost like other components Master, filer and s3 bind both the detected host ip and localhost via NewIpAndLocalListeners, but the volume server used a plain NewListener on bindIp, so a volume bound to the host ip never answers on loopback. Any tooling that polls localhost (e.g. the java test workflow's readiness curl on localhost:8080/status) then waits out its full timeout even though the volume is up and registered with the master. Use NewIpAndLocalListeners for the volume grpc, public and cluster http listeners, and add a small multiHttpdownServer composite so graceful shutdown still covers both listeners. * volume: stop and wait dual listeners concurrently A serial multiHttpdownServer.Stop let the localhost listener keep accepting requests while the bound-ip listener drained, and a serial Wait hid a localhost serving failure until the bound-ip listener ended. Run both waits and both stops in parallel: Wait reports the first error promptly, Stop closes every listener when shutdown begins. --- weed/command/volume.go | 60 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 53 insertions(+), 7 deletions(-) diff --git a/weed/command/volume.go b/weed/command/volume.go index 4646d9130a..e3244bf473 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -573,15 +573,54 @@ func (v VolumeServerOptions) isSeparatedPublicPort() bool { return *v.publicPort != *v.port } +// multiHttpdownServer gracefully stops an http server serving on both the +// bound ip and the localhost listeners as if it were a single server. +type multiHttpdownServer []httpdown.Server + +// Wait returns the first member's non-nil error as soon as it happens, so a +// failing localhost listener is reported even while the bound-ip listener +// keeps serving, and nil once every member has finished cleanly. +func (m multiHttpdownServer) Wait() error { + errs := make(chan error, len(m)) + for _, s := range m { + go func(s httpdown.Server) { errs <- s.Wait() }(s) + } + for range m { + if err := <-errs; err != nil { + return err + } + } + return nil +} + +// Stop stops all members concurrently so the localhost listener closes as +// soon as shutdown begins instead of after the bound-ip listener drains. +func (m multiHttpdownServer) Stop() error { + errs := make(chan error, len(m)) + for _, s := range m { + go func(s httpdown.Server) { errs <- s.Stop() }(s) + } + var firstErr error + for range m { + if err := <-errs; err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server { grpcPort := *v.portGrpc - grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0) + grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*v.bindIp, grpcPort, 0) if err != nil { glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err) } grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume")) volume_server_pb.RegisterVolumeServerServer(grpcS, vs) reflection.Register(grpcS) + if grpcLocalL != nil { + go grpcS.Serve(grpcLocalL) + } go func() { if err := grpcS.Serve(grpcL); err != nil { glog.Fatalf("start gRPC service failed, %s", err) @@ -594,13 +633,16 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdown.Server { publicListeningAddress := util.JoinHostPort(*v.bindIp, *v.publicPort) glog.V(0).Infoln("Start Seaweed volume server", version.Version(), "public at", publicListeningAddress) - publicListener, e := util.NewListener(publicListeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) + publicListener, publicLocalListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.publicPort, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { glog.Fatalf("Volume server listener error:%v", e) } pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute} - publicHttpDown := pubHttp.Serve(&http.Server{Handler: handler}, publicListener) + publicHttpDown := multiHttpdownServer{pubHttp.Serve(&http.Server{Handler: handler}, publicListener)} + if publicLocalListener != nil { + publicHttpDown = append(publicHttpDown, pubHttp.Serve(&http.Server{Handler: handler}, publicLocalListener)) + } go func() { if err := publicHttpDown.Wait(); err != nil { glog.Errorf("public http down wait failed, %v", err) @@ -627,7 +669,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http listeningAddress := util.JoinHostPort(*v.bindIp, *v.port) glog.V(0).Infof("Start Seaweed volume server %s at %s", version.Version(), listeningAddress) - listener, e := util.NewListener(listeningAddress, time.Duration(*v.idleConnectionTimeout)*time.Second) + listener, localListener, e := util.NewIpAndLocalListeners(*v.bindIp, *v.port, time.Duration(*v.idleConnectionTimeout)*time.Second) if e != nil { glog.Fatalf("Volume server listener error:%v", e) } @@ -659,11 +701,15 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http httpS.TLSConfig.GetCertificate = getCert } - clusterHttpServer := httpDown.Serve(httpS, listener) + clusterHttpServers := multiHttpdownServer{httpDown.Serve(httpS, listener)} + if localListener != nil { + localHttpS := &http.Server{Handler: handler, TLSConfig: httpS.TLSConfig} + clusterHttpServers = append(clusterHttpServers, httpDown.Serve(localHttpS, localListener)) + } go func() { - if e := clusterHttpServer.Wait(); e != nil { + if e := clusterHttpServers.Wait(); e != nil { glog.Fatalf("Volume server fail to serve: %v", e) } }() - return clusterHttpServer, closeCert + return clusterHttpServers, closeCert } From 6c87fb70f912e2923918c1e2aa7ed8ed16b6d950 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 16:02:33 -0700 Subject: [PATCH 17/70] server: skip local unix sockets that cannot be bound (#12000) server: skip default local unix sockets that cannot be bound weed server fatally exited during S3, filer, or SFTP startup when the default /tmp/seaweedfs--.sock path could not be removed or bound. On shared hosts (e.g. CI runners where another user's weed left a socket in the sticky /tmp directory) a stale socket file turns into 'operation not permitted' and kills the whole server, even though the TCP listeners are unaffected. Warn and skip the unix socket listener instead, matching how the gRPC local socket already degrades. An explicitly configured -localSocket path still fails fast, since a co-located S3 server may be configured to dial the filer through it. --- weed/command/filer.go | 20 ++++++++++++++------ weed/command/s3.go | 28 ++++++++++++++++++---------- weed/command/sftp.go | 16 ++++++++++++---- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 245182117a..7a9c1300f2 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -525,19 +525,27 @@ func (fo *FilerOptions) startFiler() { var socketServer *http.Server if runtime.GOOS != "windows" { localSocket := *fo.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port) } if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if explicit { + glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + } + glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) } filerSocketListener, err := net.Listen("unix", localSocket) if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + if explicit { + glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + } + glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) + } else { + socketServer = newHttpServer(defaultHandler, nil) + httpServers = append(httpServers, socketServer) + go socketServer.Serve(filerSocketListener) } - socketServer = newHttpServer(defaultHandler, nil) - httpServers = append(httpServers, socketServer) - go socketServer.Serve(filerSocketListener) } if viper.GetString("https.filer.key") != "" { diff --git a/weed/command/s3.go b/weed/command/s3.go index 958afb8eab..a37d7c8179 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -429,23 +429,31 @@ func (s3opt *S3Options) startS3Server() bool { if runtime.GOOS != "windows" { localSocket := *s3opt.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port) } if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if explicit { + glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + } + glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) } s3SocketListener, err := net.Listen("unix", localSocket) if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - socketServer := newHttpServer(router, nil) - httpServers = append(httpServers, socketServer) - go func() { - if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { - glog.Fatalf("Failed to start S3 http server: %v", err) + if explicit { + glog.Fatalf("Failed to listen on %s: %v", localSocket, err) } - }() + glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) + } else { + socketServer := newHttpServer(router, nil) + httpServers = append(httpServers, socketServer) + go func() { + if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { + glog.Errorf("Failed to start S3 http server on %s: %v", localSocket, err) + } + }() + } } listenAddress := fmt.Sprintf("%s:%d", *s3opt.bindIp, *s3opt.port) diff --git a/weed/command/sftp.go b/weed/command/sftp.go index ed4a28b79c..054096392d 100644 --- a/weed/command/sftp.go +++ b/weed/command/sftp.go @@ -180,20 +180,28 @@ func (sftpOpt *SftpOptions) startSftpServer() bool { // Set up Unix socket if on non-Windows platforms if runtime.GOOS != "windows" { localSocket := *sftpOpt.localSocket - if localSocket == "" { + explicit := localSocket != "" + if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port) } if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + if explicit { + glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) + } + glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) } go func() { // start on local unix socket sftpSocketListener, err := net.Listen("unix", localSocket) if err != nil { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + if explicit { + glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + } + glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) + return } if err := service.Serve(sftpSocketListener); err != nil { - glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err) + glog.Errorf("Failed to serve SFTP on socket %s: %v", localSocket, err) } }() } From cd97c93deb7a5ad2df8261a69e9219505cdf21e3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 16:07:32 -0700 Subject: [PATCH 18/70] command: keep filer, s3, and sftp up when the local unix socket cannot bind (#11999) * pb: share the stale-socket tolerant unix listener ServeGrpcOnLocalSocket already treats the local socket as optional; pull the remove-and-listen block out so the filer, s3, and sftp http sockets can use it too. * command: keep filer, s3, and sftp up when the local socket cannot bind A stale socket file owned by another uid (e.g. left in /tmp by a containerized job) made os.Remove fail with EPERM and the Fatalf took the whole server down. The gRPC local socket already treats the unix listener as optional; do the same for the http/sftp sockets. * filer: clear shared localSocket flag when the unix socket cannot bind s3.localFilerSocket and (in weed server) the sftp local socket share the filer's -localSocket flag pointer. If the filer cannot bind the socket it now keeps running without it, so consumers still holding the configured path would dial a dead or foreign-owned socket. * command: keep sftp on its own localSocket flag under weed server sftpOptions.localSocket was overwritten with the filer's -localSocket flag, so -sftp.localSocket was ignored and an explicit filer socket path made the sftp server try to bind the same socket the filer already held. * s3: fall back to TCP when the filer local socket cannot be dialed The embedded S3 server copies the filer's -localSocket flag before the filer tries to bind it, so a socket that never comes up left every filer HTTP request dialing a dead path. Try the unix socket first and fall back to the request's TCP address on dial failure. * filer: keep the shared localSocket flag read-only after startup Clearing *fo.localSocket raced with the embedded s3 goroutine reading the same flag pointer and could not help a reader that had already copied it. The s3 filer client's TCP fallback covers the unbound-socket case without mutating shared flag state. --- weed/command/filer.go | 18 ++---------------- weed/command/s3.go | 17 ++--------------- weed/command/server.go | 1 - weed/command/sftp.go | 28 +++++++--------------------- weed/pb/grpc_client_server.go | 25 ++++++++++++++++++------- weed/s3api/s3api_server.go | 8 ++++++-- 6 files changed, 35 insertions(+), 62 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 7a9c1300f2..5be17a26af 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "net" "net/http" "os" "runtime" @@ -522,27 +521,14 @@ func (fo *FilerOptions) startFiler() { } stopGrpcServer := func() { gracefulStopGrpc(grpcS, gracefulTimeout) } - var socketServer *http.Server if runtime.GOOS != "windows" { localSocket := *fo.localSocket explicit := localSocket != "" if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-filer-%d.sock", *fo.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - filerSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - } else { - socketServer = newHttpServer(defaultHandler, nil) + if filerSocketListener := pb.ListenOnLocalSocket(localSocket); filerSocketListener != nil { + socketServer := newHttpServer(defaultHandler, nil) httpServers = append(httpServers, socketServer) go socketServer.Serve(filerSocketListener) } diff --git a/weed/command/s3.go b/weed/command/s3.go index a37d7c8179..e2d377c64e 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -6,7 +6,6 @@ import ( "crypto/x509" "fmt" "io/ioutil" - "net" "net/http" "os" "runtime" @@ -433,24 +432,12 @@ func (s3opt *S3Options) startS3Server() bool { if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-s3-%d.sock", *s3opt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - s3SocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) - } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - } else { + if s3SocketListener := pb.ListenOnLocalSocket(localSocket); s3SocketListener != nil { socketServer := newHttpServer(router, nil) httpServers = append(httpServers, socketServer) go func() { if err := socketServer.Serve(s3SocketListener); err != nil && err != http.ErrServerClosed { - glog.Errorf("Failed to start S3 http server on %s: %v", localSocket, err) + glog.Fatalf("Failed to start S3 http server: %v", err) } }() } diff --git a/weed/command/server.go b/weed/command/server.go index a9b27bc064..5c800b3004 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -422,7 +422,6 @@ func runServer(cmd *Command, args []string) bool { if *isStartingSftp { go func() { time.Sleep(2 * time.Second) - sftpOptions.localSocket = filerOptions.localSocket sftpOptions.startSftpServer() }() } diff --git a/weed/command/sftp.go b/weed/command/sftp.go index 054096392d..eb482d018b 100644 --- a/weed/command/sftp.go +++ b/weed/command/sftp.go @@ -3,8 +3,6 @@ package command import ( "context" "fmt" - "net" - "os" "runtime" "time" @@ -184,26 +182,14 @@ func (sftpOpt *SftpOptions) startSftpServer() bool { if !explicit { localSocket = fmt.Sprintf("/tmp/seaweedfs-sftp-%d.sock", *sftpOpt.port) } - if err := os.Remove(localSocket); err != nil && !os.IsNotExist(err) { - if explicit { - glog.Fatalf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - glog.Warningf("Failed to remove %s, error: %s", localSocket, err.Error()) - } - go func() { - // start on local unix socket - sftpSocketListener, err := net.Listen("unix", localSocket) - if err != nil { - if explicit { - glog.Fatalf("Failed to listen on %s: %v", localSocket, err) + if sftpSocketListener := pb.ListenOnLocalSocket(localSocket); sftpSocketListener != nil { + go func() { + // start on local unix socket + if err := service.Serve(sftpSocketListener); err != nil { + glog.Fatalf("Failed to serve SFTP on socket %s: %v", localSocket, err) } - glog.Errorf("Failed to listen on %s, skipping local socket server: %v", localSocket, err) - return - } - if err := service.Serve(sftpSocketListener); err != nil { - glog.Errorf("Failed to serve SFTP on socket %s: %v", localSocket, err) - } - }() + }() + } } // Start the SFTP service on TCP diff --git a/weed/pb/grpc_client_server.go b/weed/pb/grpc_client_server.go index 8ba3a16f33..2a119c075e 100644 --- a/weed/pb/grpc_client_server.go +++ b/weed/pb/grpc_client_server.go @@ -140,6 +140,22 @@ func resolveLocalGrpcSocket(address string) string { return localGrpcSockets[port] } +// ListenOnLocalSocket binds a unix socket listener for a local fast path +// next to the TCP listener. A stale socket file is removed first; a path +// that cannot be cleared or bound is skipped, since the socket is optional. +func ListenOnLocalSocket(socketPath string) net.Listener { + if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { + glog.Warningf("Failed to remove old socket %s: %v", socketPath, err) + } + lc := net.ListenConfig{Control: setLocalSocketBuffers} + listener, err := lc.Listen(context.Background(), "unix", socketPath) + if err != nil { + glog.Errorf("Failed to listen on Unix socket %s: %v", socketPath, err) + return nil + } + return listener +} + // ServeGrpcOnLocalSocket starts serving a gRPC server on a Unix socket // if one is registered for the given port. func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { @@ -147,13 +163,8 @@ func ServeGrpcOnLocalSocket(grpcServer *grpc.Server, grpcPort int) { if socketPath == "" { return } - if err := os.Remove(socketPath); err != nil && !os.IsNotExist(err) { - glog.Warningf("Failed to remove old gRPC socket %s: %v", socketPath, err) - } - lc := net.ListenConfig{Control: setLocalSocketBuffers} - listener, err := lc.Listen(context.Background(), "unix", socketPath) - if err != nil { - glog.Errorf("Failed to listen on gRPC Unix socket %s: %v", socketPath, err) + listener := ListenOnLocalSocket(socketPath) + if listener == nil { return } listener = &localSocketListener{Listener: listener} diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 86b1025081..c10e311e3d 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -507,8 +507,12 @@ func NewS3ApiServerWithStore(router *mux.Router, option *S3ApiServerOption, expl } else { s3ApiServer.client = &http.Client{ Transport: &http.Transport{ - DialContext: func(_ context.Context, _, _ string) (net.Conn, error) { - return net.Dial("unix", option.LocalFilerSocket) + DialContext: func(ctx context.Context, _, addr string) (net.Conn, error) { + conn, err := (&net.Dialer{}).DialContext(ctx, "unix", option.LocalFilerSocket) + if err != nil { + return util.OutboundDialContext(ctx, "tcp", addr) + } + return conn, nil }, }, } From 8b26da8c532d662b355feb46d7fa51f6a1a3cbb5 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 16:25:23 -0700 Subject: [PATCH 19/70] ci: gate go/java/python tool caches to github-hosted runners (#12001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #11985/#11986: setup-go/setup-java/setup-python steps added or restructured since (e.g. by #11981's shared-artifact layout) were missed. On self-hosted runners setup-go restores/saves a ~5GB module cache archive per job — pure waste when the module cache is already on local disk — and the restore blocks the job for minutes. --- .github/workflows/java_integration_tests.yml | 2 +- .github/workflows/java_release.yml | 2 +- .github/workflows/java_unit_tests.yml | 2 +- .github/workflows/kafka-quicktest.yml | 2 +- .github/workflows/kafka-tests.yml | 8 ++++---- .github/workflows/posix-compliance.yml | 7 +++++-- .github/workflows/rust-volume-server-tests.yml | 1 + .github/workflows/s3-parquet-tests.yml | 6 +++--- .github/workflows/s3-snowflake-tests.yml | 2 +- .github/workflows/spark-integration-tests.yml | 2 +- .github/workflows/star_history.yml | 2 +- .github/workflows/volume-server-integration-tests.yml | 1 + 12 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml index aa8a93c9e4..363c424768 100644 --- a/.github/workflows/java_integration_tests.yml +++ b/.github/workflows/java_integration_tests.yml @@ -43,7 +43,7 @@ jobs: with: java-version: ${{ matrix.java }} distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Build SeaweedFS run: | diff --git a/.github/workflows/java_release.yml b/.github/workflows/java_release.yml index 282c56245b..3cde5af868 100644 --- a/.github/workflows/java_release.yml +++ b/.github/workflows/java_release.yml @@ -77,7 +77,7 @@ jobs: with: java-version: '17' distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} server-id: central server-username: MAVEN_CENTRAL_USERNAME server-password: MAVEN_CENTRAL_PASSWORD diff --git a/.github/workflows/java_unit_tests.yml b/.github/workflows/java_unit_tests.yml index c34f49b687..ed29f8dab3 100644 --- a/.github/workflows/java_unit_tests.yml +++ b/.github/workflows/java_unit_tests.yml @@ -34,7 +34,7 @@ jobs: with: java-version: ${{ matrix.java }} distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Build and Install SeaweedFS Client working-directory: other/java/client diff --git a/.github/workflows/kafka-quicktest.yml b/.github/workflows/kafka-quicktest.yml index 165028635a..f48c5487df 100644 --- a/.github/workflows/kafka-quicktest.yml +++ b/.github/workflows/kafka-quicktest.yml @@ -43,7 +43,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go diff --git a/.github/workflows/kafka-tests.yml b/.github/workflows/kafka-tests.yml index 8fb5cb5a36..4a59cb6724 100644 --- a/.github/workflows/kafka-tests.yml +++ b/.github/workflows/kafka-tests.yml @@ -151,7 +151,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -330,7 +330,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -492,7 +492,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go @@ -650,7 +650,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | **/go.sum id: go diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml index 973fc564bb..25fbfe5096 100644 --- a/.github/workflows/posix-compliance.yml +++ b/.github/workflows/posix-compliance.yml @@ -62,7 +62,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} - cache: true + cache: ${{ runner.environment == 'github-hosted' }} cache-dependency-path: | go.sum test/fuse_integration/go.sum @@ -568,7 +568,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Install macFUSE run: | @@ -621,6 +621,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install security analysis tools run: | @@ -677,6 +678,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | @@ -729,6 +731,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ${{ env.GO_VERSION }} + cache: ${{ runner.environment == 'github-hosted' }} - name: Install dependencies run: | diff --git a/.github/workflows/rust-volume-server-tests.yml b/.github/workflows/rust-volume-server-tests.yml index 84b80de66d..e6223aef29 100644 --- a/.github/workflows/rust-volume-server-tests.yml +++ b/.github/workflows/rust-volume-server-tests.yml @@ -65,6 +65,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable diff --git a/.github/workflows/s3-parquet-tests.yml b/.github/workflows/s3-parquet-tests.yml index 20403a533d..d0a99b04be 100644 --- a/.github/workflows/s3-parquet-tests.yml +++ b/.github/workflows/s3-parquet-tests.yml @@ -46,13 +46,13 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v7 with: python-version: ${{ matrix.python-version }} - cache: 'pip' + cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }} cache-dependency-path: 'test/s3/parquet/requirements.txt' - name: Install system dependencies @@ -154,7 +154,7 @@ jobs: uses: actions/setup-go@v7 with: go-version: ^1.26 - cache: true + cache: ${{ runner.environment == 'github-hosted' }} - name: Run Go unit tests run: | diff --git a/.github/workflows/s3-snowflake-tests.yml b/.github/workflows/s3-snowflake-tests.yml index 3993a112a0..94163e36cd 100644 --- a/.github/workflows/s3-snowflake-tests.yml +++ b/.github/workflows/s3-snowflake-tests.yml @@ -68,7 +68,7 @@ jobs: with: java-version: '17' distribution: 'temurin' - cache: 'maven' + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Install SeaweedFS run: | diff --git a/.github/workflows/spark-integration-tests.yml b/.github/workflows/spark-integration-tests.yml index ee211a1fb4..8aec50f8bd 100644 --- a/.github/workflows/spark-integration-tests.yml +++ b/.github/workflows/spark-integration-tests.yml @@ -42,7 +42,7 @@ jobs: with: java-version: '11' distribution: 'temurin' - cache: maven + cache: ${{ runner.environment == 'github-hosted' && 'maven' || '' }} - name: Set up Go uses: actions/setup-go@v7 diff --git a/.github/workflows/star_history.yml b/.github/workflows/star_history.yml index 2c87130703..016dd86c63 100644 --- a/.github/workflows/star_history.yml +++ b/.github/workflows/star_history.yml @@ -32,7 +32,7 @@ jobs: uses: actions/setup-python@v7 with: python-version: "3.x" - cache: pip + cache: ${{ runner.environment == 'github-hosted' && 'pip' || '' }} cache-dependency-path: .github/scripts/star_history.py - name: Install matplotlib diff --git a/.github/workflows/volume-server-integration-tests.yml b/.github/workflows/volume-server-integration-tests.yml index fcaf1ce976..b1a4569111 100644 --- a/.github/workflows/volume-server-integration-tests.yml +++ b/.github/workflows/volume-server-integration-tests.yml @@ -85,6 +85,7 @@ jobs: uses: actions/setup-go@v7 with: go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} - name: Download binary uses: actions/download-artifact@v7 From 168789d5f3c5077a5a8d493df34d18be2ac94f9f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 17:04:36 -0700 Subject: [PATCH 20/70] master: cap writable volumes per collection via -maxWritableVolumes (#11923) * master: cap writable volumes per collection via -maxWritableVolumes Some deployments serialize disk writes (e.g. an Unraid array where concurrent writes to multiple disks collapse throughput). With more than one writable volume in a collection, the master's weighted pick spreads assignments over all of them, so several disks get written at once. -maxWritableVolumes=N narrows the pick candidates to the N fullest writable volumes, so a collection's writes concentrate on one (or few) volumes at a time. 0 keeps the current spread-everywhere behavior. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: give master.follower a maxWritableVolumes default * master: cap writable volumes inside the requested location * master: pick the capped volumes without sorting the writable list * master: assert the capped candidate set instead of random pick outcomes * master: describe -maxWritableVolumes as a per-layout cap --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/master.go | 3 + weed/command/master_follower.go | 1 + weed/command/mini.go | 1 + weed/command/server.go | 1 + weed/server/master_server.go | 2 + weed/topology/collection.go | 5 +- weed/topology/topology.go | 14 +- weed/topology/volume_layout.go | 65 ++++++++- .../volume_layout_max_writable_test.go | 127 ++++++++++++++++++ 9 files changed, 213 insertions(+), 6 deletions(-) create mode 100644 weed/topology/volume_layout_max_writable_test.go diff --git a/weed/command/master.go b/weed/command/master.go index 81ffa6db94..2a5a409678 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -57,6 +57,7 @@ type MasterOptions struct { fileSizeLimitMB *int volumePreallocate *bool maxParallelVacuumPerServer *int + maxWritableVolumes *int vacuumIntervalSeconds *int vacuumDeleteEmptyAfterSeconds *int // pulseSeconds *int @@ -95,6 +96,7 @@ func init() { m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)") m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.") m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server") + m.maxWritableVolumes = cmdMaster.Flag.Int("maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") // m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") @@ -477,6 +479,7 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp FileSizeLimitMB: *m.fileSizeLimitMB, VolumePreallocate: *m.volumePreallocate, MaxParallelVacuumPerServer: *m.maxParallelVacuumPerServer, + MaxWritableVolumes: *m.maxWritableVolumes, VacuumIntervalSeconds: *m.vacuumIntervalSeconds, VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds, // PulseSeconds: *m.pulseSeconds, diff --git a/weed/command/master_follower.go b/weed/command/master_follower.go index 4e0983eb96..95c065775e 100644 --- a/weed/command/master_follower.go +++ b/weed/command/master_follower.go @@ -47,6 +47,7 @@ func init() { mf.metricsIntervalSec = aws.Int(0) mf.raftResumeState = aws.Bool(false) mf.maxParallelVacuumPerServer = aws.Int(1) + mf.maxWritableVolumes = aws.Int(0) mf.vacuumIntervalSeconds = aws.Int(840) mf.vacuumDeleteEmptyAfterSeconds = aws.Int(0) mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect") diff --git a/weed/command/mini.go b/weed/command/mini.go index 9fd8d0963f..381e438e9c 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -446,6 +446,7 @@ func initMiniMasterFlags() { miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)") miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + miniMasterOptions.maxWritableVolumes = cmdMini.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") diff --git a/weed/command/server.go b/weed/command/server.go index 5c800b3004..0277ccf421 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -98,6 +98,7 @@ func init() { masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + masterOptions.maxWritableVolumes = cmdServer.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") diff --git a/weed/server/master_server.go b/weed/server/master_server.go index b36679ebcd..6a5c5e7ac9 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -54,6 +54,7 @@ type MasterOption struct { FileSizeLimitMB int VolumePreallocate bool MaxParallelVacuumPerServer int + MaxWritableVolumes int VacuumIntervalSeconds int VacuumDeleteEmptyAfterSeconds int // PulseSeconds int @@ -162,6 +163,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se glog.Fatalf("create sequencer failed.") } ms.Topo = topology.NewTopology("topo", seq, uint64(ms.option.VolumeSizeLimitMB)*1024*1024, 5, replicationAsMin) + ms.Topo.SetMaxWritableVolumes(ms.option.MaxWritableVolumes) ms.Topo.SetGrowthFreezeConfig( v.GetBool("master.volume_growth.freeze_on_topology_mismatch"), v.GetDuration("master.volume_growth.state_loss_recovery_wait")) diff --git a/weed/topology/collection.go b/weed/topology/collection.go index 9fe355a083..254c951dc7 100644 --- a/weed/topology/collection.go +++ b/weed/topology/collection.go @@ -14,6 +14,7 @@ type Collection struct { Name string volumeSizeLimit uint64 replicationAsMin bool + maxWritableVolumes int storageType2VolumeLayout *util.ConcurrentReadMap } @@ -40,7 +41,9 @@ func (c *Collection) GetOrCreateVolumeLayout(rp *super_block.ReplicaPlacement, t keyString += string(diskType) } vl := c.storageType2VolumeLayout.Get(keyString, func() interface{} { - return NewVolumeLayout(rp, ttl, diskType, c.volumeSizeLimit, c.replicationAsMin) + layout := NewVolumeLayout(rp, ttl, diskType, c.volumeSizeLimit, c.replicationAsMin) + layout.maxWritableVolumes = c.maxWritableVolumes + return layout }) return vl.(*VolumeLayout) } diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 3c68d3586b..cee0018584 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -50,6 +50,7 @@ type Topology struct { volumeSizeLimit uint64 replicationAsMin bool + maxWritableVolumes int vacuumDisabledByOperator atomic.Bool // true when operator manually disables vacuum vacuumDisabledByPlugin atomic.Bool // true when disabled by the vacuum plugin monitor adminServerConnectedFunc func() bool // optional callback to check admin server presence @@ -532,10 +533,20 @@ func (t *Topology) PickForWrite(requestedCount uint64, option *VolumeGrowOption, func (t *Topology) GetVolumeLayout(collectionName string, rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType types.DiskType) *VolumeLayout { return t.collectionMap.Get(collectionName, func() interface{} { - return NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin) + c := NewCollection(collectionName, t.volumeSizeLimit, t.replicationAsMin) + c.maxWritableVolumes = t.maxWritableVolumes + return c }).(*Collection).GetOrCreateVolumeLayout(rp, ttl, diskType) } +// SetMaxWritableVolumes caps how many writable volumes in a volume layout +// (collection + replica placement + ttl + disk type) assignments are spread +// over; 0 means every writable volume is a candidate. It only affects +// collections created afterwards, so call it before serving. +func (t *Topology) SetMaxWritableVolumes(n int) { + t.maxWritableVolumes = n +} + // FindVolumeLayout is the read-only counterpart of GetVolumeLayout: it returns // nil when the collection or its layout does not exist, instead of creating // one. Lets callers check a layout's state without polluting the collection @@ -797,7 +808,6 @@ func ttlEqual(a, b *needle.TTL) bool { return a.ToUint32() == b.ToUint32() } - // findDuplicateVolumeLayouts reports every layout that already holds v.Id on // data nodes other than dn. Volume ids are global, so the same id held by two // nodes under two layouts means the allocator reused an id. diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 4219c8eff3..b0a48e750b 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -68,8 +68,11 @@ type VolumeLayout struct { deletingVolumes map[needle.VolumeId]struct{} volumeSizeLimit uint64 replicationAsMin bool - accessLock sync.RWMutex - sizeTracking map[needle.VolumeId]*volumeSizeTracking + // maxWritableVolumes caps how many writables PickForWrite spreads over; + // 0 means every writable volume is a candidate. + maxWritableVolumes int + accessLock sync.RWMutex + sizeTracking map[needle.VolumeId]*volumeSizeTracking // dropped: the layout went away with its collection; late registrations must re-resolve. dropped bool } @@ -646,13 +649,40 @@ func (vl *VolumeLayout) PickForWrite(count uint64, option *VolumeGrowOption) (vi return 0, 0, nil, true, fmt.Errorf("%s", NoWritableVolumes) } if option.DataCenter == "" && option.Rack == "" && option.DataNode == "" { - vid, locationList = vl.pickWeightedByRemaining(vl.writables) + writables := vl.writables + if vl.maxWritableVolumes > 0 && len(writables) > vl.maxWritableVolumes { + writables = vl.fullestWritable(writables) + } + vid, locationList = vl.pickWeightedByRemaining(writables) if locationList == nil || len(locationList.list) == 0 { return 0, 0, nil, false, fmt.Errorf("Strangely vid %s is on no machine!", vid.String()) } return vid, count, locationList.Copy(), false, nil } + if vl.maxWritableVolumes > 0 { + // The cap must hold inside the requested location, so collect every + // matching writable before narrowing to the fullest. + var matches []needle.VolumeId + for _, writableVolumeId := range vl.writables { + volumeLocationList := vl.vid2location[writableVolumeId] + for _, dn := range volumeLocationList.list { + if dn.matchesLocation(option) { + matches = append(matches, writableVolumeId) + break + } + } + } + if len(matches) == 0 { + return vid, count, locationList, true, fmt.Errorf("%s in DataCenter:%v Rack:%v DataNode:%v", NoWritableVolumes, option.DataCenter, option.Rack, option.DataNode) + } + if len(matches) > vl.maxWritableVolumes { + matches = vl.fullestWritable(matches) + } + vid, locationList = vl.weightedPick(matches) + return vid, count, locationList.Copy(), false, nil + } + // Scan from a random offset to collect up to pickSampleSize matching // candidates, avoiding a full scan + allocation in the common case. var sample [pickSampleSize]needle.VolumeId @@ -700,6 +730,35 @@ func (vl *VolumeLayout) pickWeightedByRemaining(candidates []needle.VolumeId) (n return vl.weightedPick(sample[:]) } +// fullestWritable narrows candidates to the maxWritableVolumes fullest ones, +// so assignments concentrate on the fullest writable volume instead of +// spreading across many disks at once. The cap is expected to be small, so the +// top-n are kept by insertion instead of sorting the whole candidate list. +func (vl *VolumeLayout) fullestWritable(candidates []needle.VolumeId) []needle.VolumeId { + n := vl.maxWritableVolumes + top := make([]needle.VolumeId, 0, n) + remaining := make([]uint64, 0, n) + for _, vid := range candidates { + r := vl.remainingSize(vid) + i := len(top) + for i > 0 && (remaining[i-1] > r || (remaining[i-1] == r && top[i-1] > vid)) { + i-- + } + if i == n { + continue + } + top = append(top, 0) + remaining = append(remaining, 0) + copy(top[i+1:], top[i:]) + copy(remaining[i+1:], remaining[i:]) + top[i], remaining[i] = vid, r + if len(top) > n { + top, remaining = top[:n], remaining[:n] + } + } + return top +} + func (vl *VolumeLayout) weightedPick(candidates []needle.VolumeId) (needle.VolumeId, *VolumeLocationList) { if len(candidates) == 1 { vid := candidates[0] diff --git a/weed/topology/volume_layout_max_writable_test.go b/weed/topology/volume_layout_max_writable_test.go new file mode 100644 index 0000000000..26b718801e --- /dev/null +++ b/weed/topology/volume_layout_max_writable_test.go @@ -0,0 +1,127 @@ +package topology + +import ( + "slices" + "testing" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" + "github.com/seaweedfs/seaweedfs/weed/storage/types" +) + +func maxWritableLayout(t *testing.T, n int) (*Topology, *DataNode, *VolumeLayout) { + t.Helper() + topo := NewTopology("mw", nil, 32*1024, 5, false) + topo.SetMaxWritableVolumes(n) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + rp, _ := super_block.NewReplicaPlacementFromString("000") + return topo, dn, topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType) +} + +// With a cap, assignments concentrate on the fullest writable volumes so at +// most cap volumes are written in parallel. +func TestPickForWriteCapsWritableVolumes(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 1) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 1000, false), + sizeTrackingVolume(3, 500, false), + }, dn) + + for i := 0; i < 20; i++ { + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(1) { + t.Fatalf("picked volume %d, want the fullest writable 1", vid) + } + } +} + +// Once the fullest volume leaves the writable list, the next fullest takes over. +func TestPickForWriteCapFallsToNextWritable(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 1) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 1000, false), + }, dn) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, true), + sizeTrackingVolume(2, 1000, false), + }, dn) + + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(2) { + t.Fatalf("picked volume %d, want 2 after 1 went read-only", vid) + } +} + +// A cap of two still concentrates on the two fullest writables; zero disables +// the cap so every writable volume is a candidate. +func TestPickForWriteCapOfTwoAndDisabled(t *testing.T) { + topo, dn, vl := maxWritableLayout(t, 2) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), + sizeTrackingVolume(2, 20000, false), + sizeTrackingVolume(3, 500, false), + }, dn) + + // the capped candidate set is deterministic even though the pick among + // them is weighted-random + if got := vl.fullestWritable(vl.writables); !slices.Equal(got, []needle.VolumeId{1, 2}) { + t.Fatalf("capped candidates %v, want [1 2]", got) + } + for i := 0; i < 40; i++ { + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}) + if err != nil { + t.Fatalf("PickForWrite: %v", err) + } + if vid != needle.VolumeId(1) && vid != needle.VolumeId(2) { + t.Fatalf("capped pick reached volume %d, want one of 1,2", vid) + } + } + + vl.maxWritableVolumes = 0 + for i := 0; i < 40; i++ { + if _, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{}); err != nil { + t.Fatalf("PickForWrite: %v", err) + } + } +} + +// The cap applies inside the requested location: with cap 1, a write naming +// dc2 must still find dc2's writable volume even though the fullest is in dc1. +func TestPickForWriteCapWithinRequestedLocation(t *testing.T) { + topo := NewTopology("mw", nil, 32*1024, 5, false) + topo.SetMaxWritableVolumes(1) + dn1 := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, 18080, "", "", map[string]uint32{"": 100}) + dn2 := topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack2"). + GetOrCreateDataNode("127.0.0.2", 8080, 18080, "", "", map[string]uint32{"": 100}) + rp, _ := super_block.NewReplicaPlacementFromString("000") + vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.HardDriveType) + + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(1, 30000, false), // fullest, in dc1 + }, dn1) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{ + sizeTrackingVolume(2, 1000, false), // in dc2 + }, dn2) + + vid, _, _, _, err := vl.PickForWrite(1, &VolumeGrowOption{DataCenter: "dc2"}) + if err != nil { + t.Fatalf("PickForWrite dc2: %v", err) + } + if vid != needle.VolumeId(2) { + t.Fatalf("picked volume %d, want dc2's volume 2", vid) + } +} From f8579bf728cca55fe3b42e752804269828b66221 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 17:05:44 -0700 Subject: [PATCH 21/70] s3: gate POST Object acl fields behind PutObjectAcl (#11939) * s3: gate POST Object acl fields behind PutObjectAcl The POST Object handler promotes acl and x-amz-grant-* form fields onto the upload's ACL headers, and resolveFileMode honors the promoted canned ACL. The PUT path only applies such headers after an s3:PutObjectAcl grant; mirror that on POST by running the shared per-key authorization with the WriteAcp action whenever the request carries ACL headers. * s3: cover the POST Object acl gate Signed POST requests carrying an acl form field are denied for a writer without the WriteAcp action and for a writer a bucket policy denies s3:PutObjectAcl on the target key; the authorization boundary is exercised directly as well. * s3: cover x-amz-grant-* POST fields and the no-ACL case * s3: encode POST policy conditions as JSON in the acl-gate test --- weed/s3api/auth_credentials.go | 14 ++ .../s3api/s3api_object_handlers_postpolicy.go | 19 +++ ...i_object_handlers_postpolicy_authz_test.go | 132 ++++++++++++++++-- 3 files changed, 151 insertions(+), 14 deletions(-) diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index 46ff18366b..502283a944 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -3183,6 +3183,20 @@ func (iam *IdentityAccessManagement) AuthorizeObjectWrite(r *http.Request, ident return iam.authorizeObjectKeyAction(r, identity, http.MethodPut, s3_constants.ACTION_WRITE, bucket, objectKey, "") } +// AuthorizeObjectAclWrite authorizes setting an ACL on one key the request URL +// does not name: a POST Object upload promotes its acl form field and any +// x-amz-grant-* fields onto the upload's ACL headers, which the PUT path only +// applies after an s3:PutObjectAcl grant. +func (iam *IdentityAccessManagement) AuthorizeObjectAclWrite(r *http.Request, identity *Identity, bucket, objectKey string) s3err.ErrorCode { + if !iam.isEnabled() { + return s3err.ErrNone + } + if bucket == "" || objectKey == "" { + return s3err.ErrNone + } + return iam.authorizeObjectKeyAction(r, identity, http.MethodPut, s3_constants.ACTION_WRITE_ACP, bucket, objectKey, "") +} + // authorizeWithIAM authorizes requests using the IAM integration policy engine func (iam *IdentityAccessManagement) authorizeWithIAM(r *http.Request, identity *Identity, action Action, bucket string, object string) s3err.ErrorCode { ctx := r.Context() diff --git a/weed/s3api/s3api_object_handlers_postpolicy.go b/weed/s3api/s3api_object_handlers_postpolicy.go index 4fb604abbf..1e8705ba11 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy.go +++ b/weed/s3api/s3api_object_handlers_postpolicy.go @@ -144,6 +144,14 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode)) return } + // An acl or x-amz-grant-* form field rides onto the upload as ACL headers, + // which the PUT path only applies under s3:PutObjectAcl. + if postRequestsAcl(r) { + if errCode := s3a.iam.AuthorizeObjectAclWrite(r, identity, bucket, object); errCode != s3err.ErrNone { + s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode)) + return + } + } versioningState, err := s3a.getVersioningState(bucket) if err != nil { @@ -275,6 +283,17 @@ func applyPostPolicyFormHeaders(r *http.Request, formValues http.Header) { } } +// postRequestsAcl reports whether the request carries ACL headers, whether +// promoted from the POST form fields or supplied on the request itself. +func postRequestsAcl(r *http.Request) bool { + for _, header := range []string{s3_constants.AmzCannedAcl, s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp} { + if r.Header.Get(header) != "" { + return true + } + } + return false +} + // Extract form fields and file data from a HTTP POST Policy func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser, fileName, fileContentType string, fileSize int64, formValues http.Header, err error) { // / HTML Form values diff --git a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go index 1dc6365071..553ff2f0a5 100644 --- a/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go +++ b/weed/s3api/s3api_object_handlers_postpolicy_authz_test.go @@ -3,10 +3,12 @@ package s3api import ( "bytes" "encoding/base64" + "encoding/json" "fmt" "mime/multipart" "net/http" "net/http/httptest" + "sort" "sync" "testing" "time" @@ -26,6 +28,10 @@ const ( postAuthzService = "s3" postAuthzBucket = "testbucket" postAuthzPrincipal = "arn:aws:iam::000000000000:user/tester" + + postAclAccessKey = "AKIAACLTESTTEST" + postAclSecretKey = "secret-key-for-acl-tests" + postAclPrincipal = "arn:aws:iam::000000000000:user/aclwriter" ) // newPostAuthzServer builds an S3ApiServer whose non-admin "tester" identity @@ -45,6 +51,11 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { Account: &iam_pb.Account{Id: "000000000000", DisplayName: "tester"}, Credentials: []*iam_pb.Credential{{AccessKey: postAuthzAccessKey, SecretKey: postAuthzSecretKey}}, Actions: []string{"Read:" + postAuthzBucket, "Write:" + postAuthzBucket, "List:" + postAuthzBucket}, + }, { + Name: "aclwriter", + Account: &iam_pb.Account{Id: "000000000000", DisplayName: "aclwriter"}, + Credentials: []*iam_pb.Credential{{AccessKey: postAclAccessKey, SecretKey: postAclSecretKey}}, + Actions: []string{"Read:" + postAuthzBucket, "Write:" + postAuthzBucket, "WriteAcp:" + postAuthzBucket}, }}, }) require.NoError(t, err) @@ -58,8 +69,14 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { "Principal": {"AWS": "%s"}, "Action": ["s3:PutObject"], "Resource": ["arn:aws:s3:::%s/denied/*"] + }, { + "Sid": "DenyAclWriterAclPrefix", + "Effect": "Deny", + "Principal": {"AWS": "%s"}, + "Action": ["s3:PutObjectAcl"], + "Resource": ["arn:aws:s3:::%s/acl-denied/*"] }] - }`, postAuthzPrincipal, postAuthzBucket)) + }`, postAuthzPrincipal, postAuthzBucket, postAclPrincipal, postAuthzBucket)) require.NoError(t, err) iam.policyEngine = policyEngine @@ -79,26 +96,42 @@ func newPostAuthzServer(t *testing.T) *S3ApiServer { // policy conditions are satisfied by the form, so the handler passes signature // and CheckPostPolicy verification and reaches the write-authorization stage. func newSignedPostRequest(t *testing.T, key string) *http.Request { + return newSignedPostRequestFor(t, key, postAuthzAccessKey, postAuthzSecretKey, nil) +} + +// extraFields are added to both the form and the policy conditions so +// CheckPostPolicy accepts them. +func newSignedPostRequestFor(t *testing.T, key, accessKey, secretKey string, extraFields map[string]string) *http.Request { t.Helper() now := time.Now().UTC() amzDate := now.Format(iso8601Format) yyyymmddStr := now.Format(yyyymmdd) - credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", postAuthzAccessKey, yyyymmddStr, postAuthzRegion, postAuthzService) + credential := fmt.Sprintf("%s/%s/%s/%s/aws4_request", accessKey, yyyymmddStr, postAuthzRegion, postAuthzService) expiration := now.Add(1 * time.Hour).Format("2006-01-02T15:04:05.000Z") - policyJSON := fmt.Sprintf( - `{"expiration":"%s","conditions":[`+ - `["eq","$bucket","%s"],`+ - `["eq","$key","%s"],`+ - `["eq","$x-amz-credential","%s"],`+ - `["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],`+ - `["eq","$x-amz-date","%s"]`+ - `]}`, - expiration, postAuthzBucket, key, credential, amzDate, - ) - encodedPolicy := base64.StdEncoding.EncodeToString([]byte(policyJSON)) - signature := getSignature(getSigningKey(postAuthzSecretKey, yyyymmddStr, postAuthzRegion, postAuthzService), encodedPolicy) + extraKeys := make([]string, 0, len(extraFields)) + for k := range extraFields { + extraKeys = append(extraKeys, k) + } + sort.Strings(extraKeys) + conditions := []any{ + []any{"eq", "$bucket", postAuthzBucket}, + []any{"eq", "$key", key}, + []any{"eq", "$x-amz-credential", credential}, + []any{"eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"}, + []any{"eq", "$x-amz-date", amzDate}, + } + for _, k := range extraKeys { + conditions = append(conditions, []any{"eq", "$" + k, extraFields[k]}) + } + policyJSON, err := json.Marshal(map[string]any{ + "expiration": expiration, + "conditions": conditions, + }) + require.NoError(t, err) + encodedPolicy := base64.StdEncoding.EncodeToString(policyJSON) + signature := getSignature(getSigningKey(secretKey, yyyymmddStr, postAuthzRegion, postAuthzService), encodedPolicy) var buf bytes.Buffer writer := multipart.NewWriter(&buf) @@ -107,6 +140,9 @@ func newSignedPostRequest(t *testing.T, key string) *http.Request { require.NoError(t, writer.WriteField("x-amz-credential", credential)) require.NoError(t, writer.WriteField("x-amz-algorithm", "AWS4-HMAC-SHA256")) require.NoError(t, writer.WriteField("x-amz-date", amzDate)) + for _, k := range extraKeys { + require.NoError(t, writer.WriteField(k, extraFields[k])) + } require.NoError(t, writer.WriteField("policy", encodedPolicy)) require.NoError(t, writer.WriteField("x-amz-signature", signature)) filePart, err := writer.CreateFormFile("file", "payload.txt") @@ -170,3 +206,71 @@ func TestAuthorizeObjectWrite(t *testing.T) { s3a.iam.AuthorizeObjectWrite(req, admin, postAuthzBucket, "denied/secret.txt"), "an admin is not confined by the bucket policy") } + +// TestPostPolicyBucketHandlerGatesAclFields drives signed POST uploads carrying +// an acl form field and asserts the handler rejects them the way the PUT path +// does: a writer without the WriteAcp action is refused, and so is a writer the +// bucket policy denies s3:PutObjectAcl on the target key. +func TestPostPolicyBucketHandlerGatesAclFields(t *testing.T) { + s3a := newPostAuthzServer(t) + + rec := httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "allowed/report.txt", + postAuthzAccessKey, postAuthzSecretKey, map[string]string{"acl": "public-read"})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an acl field needs WriteAcp like PUT, body: %s", rec.Body.String()) + + rec = httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "acl-denied/secret.txt", + postAclAccessKey, postAclSecretKey, map[string]string{"acl": "public-read"})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an acl field must honor the bucket-policy PutObjectAcl deny, body: %s", rec.Body.String()) + + for _, field := range []string{"x-amz-grant-read", "x-amz-grant-write", "x-amz-grant-read-acp", "x-amz-grant-write-acp", "x-amz-grant-full-control"} { + rec = httptest.NewRecorder() + s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "allowed/report.txt", + postAuthzAccessKey, postAuthzSecretKey, + map[string]string{field: `id="0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"`})) + assert.Equal(t, http.StatusForbidden, rec.Code, + "POST with an %s field needs WriteAcp like PUT, body: %s", field, rec.Body.String()) + } +} + +func TestPostRequestsAcl(t *testing.T) { + for _, header := range []string{ + s3_constants.AmzCannedAcl, s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, + s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp, + } { + req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil) + req.Header.Set(header, "x") + assert.True(t, postRequestsAcl(req), "%s must trip the ACL gate", header) + } + assert.False(t, postRequestsAcl(httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil))) +} + +// TestAuthorizeObjectAclWrite covers the acl authorization decision at the +// boundary, for the same reason as TestAuthorizeObjectWrite. +func TestAuthorizeObjectAclWrite(t *testing.T) { + s3a := newPostAuthzServer(t) + tester, _, found := s3a.iam.lookupByAccessKey(postAuthzAccessKey) + require.True(t, found) + aclwriter, _, found := s3a.iam.lookupByAccessKey(postAclAccessKey) + require.True(t, found) + + req := httptest.NewRequest(http.MethodPost, "/"+postAuthzBucket, nil) + + assert.Equal(t, s3err.ErrAccessDenied, + s3a.iam.AuthorizeObjectAclWrite(req, tester, postAuthzBucket, "allowed/report.txt"), + "a writer without the WriteAcp action cannot attach an ACL") + assert.Equal(t, s3err.ErrNone, + s3a.iam.AuthorizeObjectAclWrite(req, aclwriter, postAuthzBucket, "allowed/report.txt"), + "the WriteAcp holder clears keys the policy does not deny") + assert.Equal(t, s3err.ErrAccessDenied, + s3a.iam.AuthorizeObjectAclWrite(req, aclwriter, postAuthzBucket, "acl-denied/secret.txt"), + "a bucket-policy PutObjectAcl deny must block the ACL-bearing upload") + + admin := &Identity{Name: "admin", Actions: []Action{s3_constants.ACTION_ADMIN}} + assert.Equal(t, s3err.ErrNone, + s3a.iam.AuthorizeObjectAclWrite(req, admin, postAuthzBucket, "acl-denied/secret.txt"), + "an admin is not confined by the bucket policy") +} From f5e319ae8ec50bdad60cc6f7688032c6ce103720 Mon Sep 17 00:00:00 2001 From: Jay Hemnani <193022578+jayhemnani9910@users.noreply.github.com> Date: Sat, 24 Oct 2026 05:38:49 +0530 Subject: [PATCH 22/70] iam: GetUserPolicy returns NoSuchEntity for a policy name the user does not have (#11944) * iam: GetUserPolicy returns NoSuchEntity for a policy name the user does not have When the user has stored inline policies, an unknown name fell through to the policy rebuilt from the user's actions, so a deleted policy still read back. The standalone IAM server already returns NoSuchEntity here. * iam: return service failure when listing inline policies fails A ListUserInlinePolicies error in GetUserPolicy fell through to action-based reconstruction, masking a store failure as a policy document. Propagate it as ServiceFailureException instead, matching the standalone iam server and the other callers in this file. * iam: return service failure when the inline policy store read fails --------- Co-authored-by: chrislusf Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/s3api_embedded_iam.go | 12 +++++-- weed/s3api/s3api_embedded_iam_test.go | 50 +++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/weed/s3api/s3api_embedded_iam.go b/weed/s3api/s3api_embedded_iam.go index 4b837117f4..1c6aa1bec0 100644 --- a/weed/s3api/s3api_embedded_iam.go +++ b/weed/s3api/s3api_embedded_iam.go @@ -1291,9 +1291,9 @@ func (e *EmbeddedIamApi) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values ctx := context.Background() storedDoc, storeErr := e.credentialManager.GetUserInlinePolicy(ctx, userName, policyName) if storeErr != nil { - glog.Warningf("GetUserPolicy: failed to read stored inline policy %q for user %s: %v; falling back to reconstruction", policyName, userName, storeErr) + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("get inline policy %s for %s: %w", policyName, userName, storeErr)} } - if storeErr == nil && storedDoc != nil { + if storedDoc != nil { policyDocumentJSON, err := json.Marshal(storedDoc) if err != nil { return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: err} @@ -1301,6 +1301,14 @@ func (e *EmbeddedIamApi) GetUserPolicy(s3cfg *iam_pb.S3ApiConfiguration, values resp.GetUserPolicyResult.PolicyDocument = string(policyDocumentJSON) return resp, nil } + // The user has stored inline policies, but none by this name + policyNames, err := e.credentialManager.ListUserInlinePolicies(ctx, userName) + if err != nil { + return resp, &iamError{Code: iam.ErrCodeServiceFailureException, Error: fmt.Errorf("list inline policies for %s: %w", userName, err)} + } + if len(policyNames) > 0 { + return resp, &iamError{Code: iam.ErrCodeNoSuchEntityException, Error: fmt.Errorf("the user policy with name %s cannot be found", policyName)} + } } // Fallback: reconstruct from ident.Actions (lossy - fine-grained actions diff --git a/weed/s3api/s3api_embedded_iam_test.go b/weed/s3api/s3api_embedded_iam_test.go index 2e7f9206af..4269958de9 100644 --- a/weed/s3api/s3api_embedded_iam_test.go +++ b/weed/s3api/s3api_embedded_iam_test.go @@ -1979,6 +1979,56 @@ func TestEmbeddedIamGetUserPolicyFallback(t *testing.T) { assert.ElementsMatch(t, []string{"arn:aws:s3:::b-le*", "arn:aws:s3:::b-le*/*"}, stmt.Resource.Strings()) } +// TestEmbeddedIamGetUserPolicyMissingName checks that a policy name the user does not have +// is NoSuchEntity, not a policy rebuilt from the user's other inline policies. +func TestEmbeddedIamGetUserPolicyMissingName(t *testing.T) { + api := NewEmbeddedIamApiForTest() + s3cfg := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{Name: "steward"}}, + } + api.mockConfig = s3cfg + + _, iamErr := api.PutUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + "PolicyDocument": []string{`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`}, + }) + require.Nil(t, iamErr) + + _, iamErr = api.GetUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"write"}, + }) + require.NotNil(t, iamErr) + assert.Equal(t, iam.ErrCodeNoSuchEntityException, iamErr.Code) +} + +// TestEmbeddedIamGetUserPolicyStoreError checks that a store failure surfaces as +// ServiceFailure instead of rebuilding a policy from the identity's actions. +func TestEmbeddedIamGetUserPolicyStoreError(t *testing.T) { + api := NewEmbeddedIamApiForTest() + s3cfg := &iam_pb.S3ApiConfiguration{ + Identities: []*iam_pb.Identity{{Name: "steward"}}, + } + api.mockConfig = s3cfg + + _, iamErr := api.PutUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + "PolicyDocument": []string{`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`}, + }) + require.Nil(t, iamErr) + + api.credentialManager = &credential.CredentialManager{Store: &memory.MemoryStore{}} + + _, iamErr = api.GetUserPolicy(s3cfg, url.Values{ + "UserName": []string{"steward"}, + "PolicyName": []string{"read"}, + }) + require.NotNil(t, iamErr) + assert.Equal(t, iam.ErrCodeServiceFailureException, iamErr.Code) +} + // TestEmbeddedIamGetUserPolicyUserNotFound tests GetUserPolicy with non-existent user func TestEmbeddedIamGetUserPolicyUserNotFound(t *testing.T) { api := NewEmbeddedIamApiForTest() From baf0dd680e127563b4a119e28b33b491380acc0a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 17:29:36 -0700 Subject: [PATCH 23/70] ec.encode: require shard placement to meet -shardReplicaPlacement (#11947) * ec: measure shard placement against replica-placement caps ecShardPlacementViolations reports where a volume's newest-generation shard placement exceeds the per-node and per-rack caps a -shardReplicaPlacement requests, using the same digit mapping the balance planner enforces (SameRackCount = shards/node, DiffRackCount = shards/rack, dc-qualified rack keys). * ec.encode: -strictPlacement keeps originals when placement breaches caps -shardReplicaPlacement tells the balancer the requested per-node and per-rack shard limits, but nothing verifies the outcome. With -strictPlacement the pre-delete shard verification also measures the final placement against those caps: a volume still over the node or rack limit aborts before its source is deleted, so the encode only completes when the requested failure-domain spread is actually met. * ec.encode: cover the strict-placement delete gate with a test The gate in verifyEcShardsBeforeDelete now runs against a stubbed topology: a compliant spread passes while an over-cap placement is refused before source deletion. The retry knobs become package vars so the test does not wait out the heartbeat-propagation backoff. * ec.encode: enforce -shardReplicaPlacement without a flag Drop the -strictPlacement option: the requested per-node and per-rack caps are always requirements at the pre-delete verification, so the encode only completes when the placement actually honors them. --- weed/ec/ec_encode.go | 78 ++++++++++++++++++-- weed/ec/ec_encode_test.go | 124 ++++++++++++++++++++++++++++++++ weed/shell/command_ec_encode.go | 3 +- 3 files changed, 197 insertions(+), 8 deletions(-) diff --git a/weed/ec/ec_encode.go b/weed/ec/ec_encode.go index 765810af01..8fde7babc0 100644 --- a/weed/ec/ec_encode.go +++ b/weed/ec/ec_encode.go @@ -121,7 +121,7 @@ func ProcessEcEncodeBatch(env *Env, writer io.Writer, volumeIds []needle.VolumeI if err := EcBalance(env, balanceCollections, "", rp, diskType, maxParallelization, 0, applyBalancing, skippedNodes, nil, volumeIds); err != nil { return fmt.Errorf("re-balance ec shards for collection(s) %v: %w", balanceCollections, err) } - if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing); err != nil { + if err := verifyEcShardsBeforeDelete(env, volumeIds, diskType, applyBalancing, rp); err != nil { return fmt.Errorf("verify EC shards before deleting originals: %w", err) } // Past verify the EC copy is recoverable; a delete failure below must not @@ -727,6 +727,54 @@ func ecShardsClumpedOnOneNode(topoInfo *master_pb.TopologyInfo, vid needle.Volum return "", false } +// verifyEcShardAttempts and verifyEcShardRetryInterval bound how long +// verifyEcShardsBeforeDelete waits for post-rebalance shard locations to +// propagate through volume-server heartbeats before giving up. They are +// package vars so tests can shrink the wait. +var verifyEcShardAttempts = 10 +var verifyEcShardRetryInterval = 2 * time.Second + +// ecShardRackByNode maps every volume server's address to its "dc:rack" +// failure-domain key, matching how the balance planner keys racks. +func ecShardRackByNode(topoInfo *master_pb.TopologyInfo) map[pb.ServerAddress]string { + rackOf := make(map[pb.ServerAddress]string) + EachDataNode(topoInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) { + rackOf[pb.NewServerAddressFromDataNode(dn)] = fmt.Sprintf("%s:%s", dc, rack) + }) + return rackOf +} + +// ecShardPlacementViolations reports where the newest-generation placement of +// vid breaches the caps the encode asked for: the replica placement's node +// digit (SameRackCount) bounds the shards of the volume one node may hold and +// its rack digit (DiffRackCount) the shards one rack may hold. A zero digit +// leaves that level unbounded; a nil placement binds nothing. byNode counts +// only the newest encode generation -- a stale orphan's shards do not weaken +// the fresh set. +func ecShardPlacementViolations(vid needle.VolumeId, byNode map[pb.ServerAddress]erasure_coding.ShardBits, rackOf map[pb.ServerAddress]string, rp *super_block.ReplicaPlacement) []string { + if rp == nil || (rp.SameRackCount <= 0 && rp.DiffRackCount <= 0) { + return nil + } + rackCount := make(map[string]int) + var violations []string + for addr, bits := range byNode { + count := bits.Count() + if rp.SameRackCount > 0 && count > rp.SameRackCount { + violations = append(violations, fmt.Sprintf("volume %d: %d shards on node %s, over the requested %d per node", vid, count, addr, rp.SameRackCount)) + } + rackCount[rackOf[addr]] += count + } + if rp.DiffRackCount > 0 { + for rack, count := range rackCount { + if count > rp.DiffRackCount { + violations = append(violations, fmt.Sprintf("volume %d: %d shards on rack %s, over the requested %d per rack", vid, count, rack, rp.DiffRackCount)) + } + } + } + sort.Strings(violations) + return violations +} + // ecShardSummaryByNode says where a volume's shards are, one entry per node, // sorted so the message is stable. It names the ids and not just the count: a // set holding shards 0-9 and one holding 4-13 are both "10 shards", and which @@ -789,7 +837,7 @@ func ecShardSummaryByNode(byNode map[pb.ServerAddress]erasure_coding.ShardBits) return summary } -func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool) error { +func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType types.DiskType, expectSpread bool, rp *super_block.ReplicaPlacement) error { // Shard relocations from the preceding EC balance reach the master via // volume-server heartbeats, so freshly distributed shards may not all be // visible in the master topology immediately. Poll a few times before @@ -802,12 +850,16 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType // rebalance ran in apply mode), a volume whose shards all still sit on one // node while another node has free slots also aborts the deletion: losing // that node would lose the volume, so the original is the safer copy. - const maxAttempts = 10 - const retryInterval = 2 * time.Second + // The replica-placement caps the encode asked for are requirements, not + // preferences: a volume still over the node or rack limit aborts the + // deletion the same way. + maxAttempts := verifyEcShardAttempts + retryInterval := verifyEcShardRetryInterval var lastErr error var lastDegraded []string var lastClumped []string + var lastPlacement []string for attempt := 0; attempt < maxAttempts; attempt++ { topoInfo, _, err := env.FetchTopology(0) if err != nil { @@ -817,6 +869,11 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType lastErr = nil lastDegraded = lastDegraded[:0] lastClumped = lastClumped[:0] + lastPlacement = lastPlacement[:0] + var rackOf map[pb.ServerAddress]string + if rp != nil && (rp.SameRackCount > 0 || rp.DiffRackCount > 0) { + rackOf = ecShardRackByNode(topoInfo) + } for _, vid := range volumeIds { // Count the shards wherever they landed, as waitForEcShardsToRegister // above already does. generateEcShards writes them beside the source @@ -852,6 +909,10 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType continue } } + if violations := ecShardPlacementViolations(vid, byNode, rackOf, rp); len(violations) > 0 { + lastPlacement = append(lastPlacement, violations...) + continue + } if degraded { lastDegraded = append(lastDegraded, fmt.Sprintf("volume %d: %d/%d shards", vid, union.Count(), totalShards)) continue @@ -861,12 +922,12 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType vid, union.Count(), totalShards, len(byNode)) } - if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 { + if lastErr == nil && len(lastDegraded) == 0 && len(lastClumped) == 0 && len(lastPlacement) == 0 { return nil } if attempt < maxAttempts-1 { - glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v", - attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped) + glog.V(0).Infof("EC shard verification incomplete (attempt %d/%d), waiting for shard locations to propagate: %v %v %v %v", + attempt+1, maxAttempts, lastErr, lastDegraded, lastClumped, lastPlacement) time.Sleep(retryInterval) } } @@ -878,6 +939,9 @@ func verifyEcShardsBeforeDelete(env *Env, volumeIds []needle.VolumeId, diskType if len(lastClumped) > 0 { return fmt.Errorf("EC shards still sit on a single node after rebalance even though other nodes have free slots (%v); keeping the original volumes. Run ec.balance -apply, verify the spread, then delete the originals or re-run ec.encode", lastClumped) } + if len(lastPlacement) > 0 { + return fmt.Errorf("EC shard placement does not meet the requested replica placement limits (%v); keeping the original volumes. Loosen -shardReplicaPlacement or add capacity, then re-run ec.encode", lastPlacement) + } glog.Warningf("EC shard set incomplete but recoverable after %d attempts, proceeding with source deletion (rebuild missing shards with ec.rebuild): %v", maxAttempts, lastDegraded) return nil diff --git a/weed/ec/ec_encode_test.go b/weed/ec/ec_encode_test.go index feb6b6030d..2cb0790642 100644 --- a/weed/ec/ec_encode_test.go +++ b/weed/ec/ec_encode_test.go @@ -1,6 +1,7 @@ package ec import ( + "fmt" "testing" "time" @@ -8,6 +9,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/storage/needle" + "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util/wildcard" "github.com/stretchr/testify/assert" @@ -529,3 +531,125 @@ func TestCollectSourceFreeVolumeCountsRefundsClearedShards(t *testing.T) { assert.Equal(t, 1, counts["3-127.0.0.1:8110"], "no refund on a node the sweep skipped as unreachable") } + +func TestEcShardPlacementViolations(t *testing.T) { + node := func(id string, shardBits uint32) *master_pb.DataNodeInfo { + return &master_pb.DataNodeInfo{ + Id: id, + DiskInfos: map[string]*master_pb.DiskInfo{ + "": { + MaxVolumeCount: 10, + EcShardInfos: []*master_pb.VolumeEcShardInformationMessage{ + {Id: 1, EcIndexBits: shardBits, DiskId: 0}, + }, + }, + }, + } + } + topo := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{{ + Id: "dc1", + RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n1:8080", 0b0011), // shards 0,1 + node("n2:8080", 0b0100), // shard 2 + }}, + {Id: "rack2", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n3:8080", 0b1000), // shard 3 + }}, + }, + }}, + } + vid := needle.VolumeId(1) + violationsOf := func(topo *master_pb.TopologyInfo, rp *super_block.ReplicaPlacement) []string { + return ecShardPlacementViolations(vid, CollectEcShardBitsByNode(topo, vid), ecShardRackByNode(topo), rp) + } + + // No caps requested: nothing to violate. + assert.Empty(t, violationsOf(topo, nil)) + assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{})) + + // Node cap: n1 holds 2 shards, over the requested 1 per node. + violations := violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 1}) + assert.Len(t, violations, 1) + assert.Contains(t, violations[0], "per node") + + // Rack cap: rack1 holds 3 shards, over the requested 2 per rack; the + // same-named rack in another dc is a different failure domain. + topo2 := &master_pb.TopologyInfo{ + DataCenterInfos: []*master_pb.DataCenterInfo{ + {Id: "dc1", RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n1:8080", 0b0111), // shards 0,1,2 + }}, + }}, + {Id: "dc2", RackInfos: []*master_pb.RackInfo{ + {Id: "rack1", DataNodeInfos: []*master_pb.DataNodeInfo{ + node("n9:8080", 0b1000), // shard 3 + }}, + }}, + }, + } + violations = violationsOf(topo2, &super_block.ReplicaPlacement{DiffRackCount: 2}) + assert.Len(t, violations, 1) + assert.Contains(t, violations[0], "dc1:rack1") + + // Caps that fit the layout: no violations. + assert.Empty(t, violationsOf(topo, &super_block.ReplicaPlacement{SameRackCount: 2, DiffRackCount: 3})) +} + +func TestVerifyEcShardsBeforeDeletePlacement(t *testing.T) { + oldAttempts, oldInterval := verifyEcShardAttempts, verifyEcShardRetryInterval + verifyEcShardAttempts, verifyEcShardRetryInterval = 2, time.Millisecond + t.Cleanup(func() { + verifyEcShardAttempts, verifyEcShardRetryInterval = oldAttempts, oldInterval + }) + + vid := needle.VolumeId(1) + topologyWith := func(nodeShards map[string]uint32) *master_pb.TopologyInfo { + var nodes []*master_pb.DataNodeInfo + for id, bits := range nodeShards { + si := erasure_coding.NewShardsInfo() + for sid := 0; sid < erasure_coding.TotalShardsCount; sid++ { + if bits&(1< Date: Fri, 23 Oct 2026 17:52:50 -0700 Subject: [PATCH 24/70] ci: set up Go for CodeQL autobuild (#12007) The workflow relied on a Go toolchain preinstalled on the runner image; self-hosted runners without setup-go fail at autobuild with 'go: executable file not found'. --- .github/workflows/codeql.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index edc2af3c98..79adf73c98 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -25,6 +25,12 @@ jobs: - name: Checkout repository uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: 'go.mod' + cache: ${{ runner.environment == 'github-hosted' }} + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v4.39.0 From b2817e853d1ecfccee65d4df42d0bd49b915de17 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:15:01 -0700 Subject: [PATCH 25/70] test: raise mini-server startup wait from 30s to 90s in Makefiles (#12005) * test: raise mini-server startup wait from 30s to 90s in Makefiles Self-hosted/persistent runners under load regularly need >30s for weed mini (master+volume+filer+s3+iceberg+lance+webdav+worker) to become healthy; hosted runners start in <10s. Siblings already wait 90s (s3/checksum, s3/cors). Fixes the S3 Filer Group first-run flake on busy self-hosted runners. * test: align Makefile startup guards and progress totals with 90s bound Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/kms/Makefile | 8 ++++---- test/s3/copying/Makefile | 4 ++-- test/s3/cors/Makefile | 2 +- test/s3/filer_group/Makefile | 4 ++-- test/s3/iam/Makefile | 2 +- test/s3/sse/Makefile | 6 +++--- test/s3/versioning/Makefile | 2 +- test/tus/Makefile | 8 ++++---- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/kms/Makefile b/test/kms/Makefile index a46aa19b25..47736f6800 100644 --- a/test/kms/Makefile +++ b/test/kms/Makefile @@ -174,12 +174,12 @@ start-openbao-ci: ## Start OpenBao via Docker for CI ghcr.io/openbao/openbao:latest \ bao server -dev -dev-root-token-id=$(OPENBAO_TOKEN) -dev-listen-address=0.0.0.0:8200 @echo "Waiting for OpenBao to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s $(OPENBAO_ADDR)/v1/sys/health > /dev/null 2>&1; then \ echo "$(GREEN)OpenBao is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ + if [ $$i -eq 90 ]; then \ echo "$(RED)Timeout waiting for OpenBao$(NC)"; \ docker logs openbao-ci 2>&1 | tail -20; \ exit 1; \ @@ -213,12 +213,12 @@ start-seaweedfs-ci: check-binary ## Start weed mini with OpenBao KMS config -ip=127.0.0.1 \ > /tmp/seaweedfs-kms-mini.log 2>&1 & echo $$! > /tmp/weed-kms-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "$(GREEN)S3 service is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ + if [ $$i -eq 90 ]; then \ echo "$(RED)Timeout waiting for S3 service$(NC)"; \ echo "=== Last 50 lines of server log ==="; \ tail -50 /tmp/seaweedfs-kms-mini.log 2>/dev/null || echo "No server log found"; \ diff --git a/test/s3/copying/Makefile b/test/s3/copying/Makefile index ea399223f0..50bedf2007 100644 --- a/test/s3/copying/Makefile +++ b/test/s3/copying/Makefile @@ -76,12 +76,12 @@ start-seaweedfs: check-binary -ip=127.0.0.1 \ > /tmp/seaweedfs-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "$(YELLOW)Waiting for S3 service to be ready...$(NC)" - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s -f http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "$(GREEN)S3 service is ready$(NC)"; \ break; \ fi; \ - echo "Waiting for S3 service... ($$i/30)"; \ + echo "Waiting for S3 service... ($$i/90)"; \ sleep 1; \ done @sleep 2 diff --git a/test/s3/cors/Makefile b/test/s3/cors/Makefile index 1378c7439e..ac56e11bc0 100644 --- a/test/s3/cors/Makefile +++ b/test/s3/cors/Makefile @@ -92,7 +92,7 @@ start-server: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ diff --git a/test/s3/filer_group/Makefile b/test/s3/filer_group/Makefile index 21d5c050a8..7420b23149 100644 --- a/test/s3/filer_group/Makefile +++ b/test/s3/filer_group/Makefile @@ -79,14 +79,14 @@ start-server: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ fi; \ sleep 1; \ done; \ - echo "❌ Server failed to start within 30 seconds"; \ + echo "❌ Server failed to start within 90 seconds"; \ if [ -f weed-server.log ]; then \ cat weed-server.log; \ fi; \ diff --git a/test/s3/iam/Makefile b/test/s3/iam/Makefile index 6dbb54299f..70f8972e14 100644 --- a/test/s3/iam/Makefile +++ b/test/s3/iam/Makefile @@ -81,7 +81,7 @@ start-services: ## Start SeaweedFS services for testing wait-for-services: ## Wait for all services to be ready @echo "Waiting for services to be ready..." @echo "Checking S3 API server..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://localhost:$(S3_PORT) > /dev/null 2>&1; then \ echo "OK S3 API server is ready"; \ exit 0; \ diff --git a/test/s3/sse/Makefile b/test/s3/sse/Makefile index cd7d50e9e8..e5072dbdba 100644 --- a/test/s3/sse/Makefile +++ b/test/s3/sse/Makefile @@ -106,7 +106,7 @@ start-seaweedfs: check-binary > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ @@ -347,7 +347,7 @@ start-seaweedfs-ci: check-binary > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ @@ -491,7 +491,7 @@ test-volume-encryption: build-weed -ip=127.0.0.1 \ > /tmp/seaweedfs-sse-mini.log 2>&1 & echo $$! > /tmp/weed-mini.pid @echo "Checking S3 service is ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s http://127.0.0.1:$(S3_PORT) > /dev/null 2>&1; then \ echo "✅ S3 service is ready"; \ break; \ diff --git a/test/s3/versioning/Makefile b/test/s3/versioning/Makefile index 621543bbc8..f41cfe3905 100644 --- a/test/s3/versioning/Makefile +++ b/test/s3/versioning/Makefile @@ -331,7 +331,7 @@ start-server-simple: check-deps echo $$! > weed-server.pid @echo "Waiting for S3 server to be ready..." - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if echo | nc -z localhost $(S3_PORT); then \ echo "S3 server is ready!"; \ exit 0; \ diff --git a/test/tus/Makefile b/test/tus/Makefile index 172b3af506..63f2ed50cc 100644 --- a/test/tus/Makefile +++ b/test/tus/Makefile @@ -106,17 +106,17 @@ start-seaweedfs: check-binary # Wait for filer to be ready @echo "$(YELLOW)Waiting for filer to be ready...$(NC)" - @for i in $$(seq 1 30); do \ + @for i in $$(seq 1 90); do \ if curl -s -f http://127.0.0.1:$(FILER_PORT)/ > /dev/null 2>&1; then \ echo "$(GREEN)Filer is ready$(NC)"; \ break; \ fi; \ - if [ $$i -eq 30 ]; then \ - echo "$(RED)Filer failed to start within 30 seconds$(NC)"; \ + if [ $$i -eq 90 ]; then \ + echo "$(RED)Filer failed to start within 90 seconds$(NC)"; \ $(MAKE) debug-logs; \ exit 1; \ fi; \ - echo "Waiting for filer... ($$i/30)"; \ + echo "Waiting for filer... ($$i/90)"; \ sleep 1; \ done From 12d05372156ae9283ec6510059dc4753b38437f9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:18:32 -0700 Subject: [PATCH 26/70] test: gofmt s3tables test files (#12011) Unformatted test files leave the S3 Tables Format Check red on every branch until they are re-formatted. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- test/s3tables/admin/admin_iceberg_pages_test.go | 2 +- test/s3tables/maintenance/maintenance_integration_test.go | 2 +- test/s3tables/table-buckets/s3tables_integration_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/s3tables/admin/admin_iceberg_pages_test.go b/test/s3tables/admin/admin_iceberg_pages_test.go index 1cc32c2168..e914c3d2a7 100644 --- a/test/s3tables/admin/admin_iceberg_pages_test.go +++ b/test/s3tables/admin/admin_iceberg_pages_test.go @@ -155,7 +155,7 @@ func startCluster(testDir string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-webdav=false", // Disable WebDAV + "-webdav=false", // Disable WebDAV "-s3.port.lance=0", // Disable Lance "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-master.volumeSizeLimitMB=32", diff --git a/test/s3tables/maintenance/maintenance_integration_test.go b/test/s3tables/maintenance/maintenance_integration_test.go index a92ef92657..ddbfb69172 100644 --- a/test/s3tables/maintenance/maintenance_integration_test.go +++ b/test/s3tables/maintenance/maintenance_integration_test.go @@ -151,7 +151,7 @@ func startCluster(testDir string, extraArgs []string) (*testCluster, error) { "-filer.port.grpc=" + strconv.Itoa(filerGrpc), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3Grpc), - "-webdav=false", // Disable WebDAV + "-webdav=false", // Disable WebDAV "-s3.port.lance=0", // Disable Lance "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", diff --git a/test/s3tables/table-buckets/s3tables_integration_test.go b/test/s3tables/table-buckets/s3tables_integration_test.go index c99d6d4393..70d7979df1 100644 --- a/test/s3tables/table-buckets/s3tables_integration_test.go +++ b/test/s3tables/table-buckets/s3tables_integration_test.go @@ -679,7 +679,7 @@ func startMiniClusterInDir(testDir string, extraArgs []string) (*TestCluster, er "-filer.port.grpc=" + strconv.Itoa(filerGrpcPort), "-s3.port=" + strconv.Itoa(s3Port), "-s3.port.grpc=" + strconv.Itoa(s3GrpcPort), - "-webdav=false", // Disable WebDAV + "-webdav=false", // Disable WebDAV "-s3.port.lance=0", // Disable Lance "-s3.port.iceberg=" + strconv.Itoa(icebergPort), "-admin.ui=false", // Disable admin UI From 3810fb112341cfac9ff982bb9b0f9b759646036f Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Sat, 24 Oct 2026 06:51:26 +0530 Subject: [PATCH 27/70] s3: url-encode ListMultipartUploads results for encoding-type=url (#11998) * s3: url-encode ListMultipartUploads results for encoding-type=url The response echoed EncodingType=url but returned raw upload keys, and XML 1.0 cannot carry control characters, so the encoder replaced them with U+FFFD. The listed key then matched no upload: AbortMultipartUpload with it answered 404 NoSuchUpload, leaving such uploads uncleanable. Encode the key-named elements AWS encodes for this API (Key, Prefix, Delimiter, KeyMarker, NextKeyMarker) the way ListObjects encodes its keys when encoding-type=url is requested. Upload ids stay opaque, and responses without encoding-type=url are unchanged. Fixes #11991 * s3: echo EncodingType only when encoding-type=url is honored A request with an unrecognized encoding-type echoed the value back in while the keys stayed raw, matching neither AWS (which omits the element unless url encoding applies) nor the ListObjects handlers, which gate the echo on the parsed url flag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: chrislusf Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/filer_multipart.go | 28 ++++++- .../filer_multipart_list_encoding_test.go | 76 +++++++++++++++++++ weed/s3api/s3api_object_handlers_multipart.go | 2 - 3 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 weed/s3api/filer_multipart_list_encoding_test.go diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 6f22429fb6..9d9f1f8cc8 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1400,7 +1400,6 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output := &ListMultipartUploadsResult{ Bucket: input.Bucket, Delimiter: input.Delimiter, - EncodingType: input.EncodingType, KeyMarker: input.KeyMarker, MaxUploads: aws.Int64(int64(p.limit)), Prefix: input.Prefix, @@ -1425,9 +1424,36 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output.IsTruncated = aws.Bool(true) output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId } + if aws.StringValue(input.EncodingType) == s3.EncodingTypeUrl { + // AWS url-encodes these key-named elements when encoding-type=url is + // requested; without it XML 1.0 cannot carry the keys at all and the + // encoder replaces their characters with U+FFFD. + output.EncodingType = input.EncodingType + output.Delimiter = encodeMultipartUploadURLField(output.Delimiter) + output.Prefix = encodeMultipartUploadURLField(output.Prefix) + output.KeyMarker = encodeMultipartUploadURLField(output.KeyMarker) + output.NextKeyMarker = encodeMultipartUploadURLField(output.NextKeyMarker) + for _, upload := range output.Upload { + upload.Key = encodeMultipartUploadURLField(upload.Key) + } + } return output } +// encodeMultipartUploadURLField url-encodes key text the way ListObjects does: +// every "/"-separated segment is query-escaped and the separators are kept, so a +// client recovers the exact key with url.QueryUnescape. +func encodeMultipartUploadURLField(value *string) *string { + if value == nil { + return nil + } + segments := strings.Split(*value, "/") + for i, segment := range segments { + segments[i] = url.QueryEscape(segment) + } + return aws.String(strings.Join(segments, "/")) +} + type ListPartsResult struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult"` diff --git a/weed/s3api/filer_multipart_list_encoding_test.go b/weed/s3api/filer_multipart_list_encoding_test.go new file mode 100644 index 0000000000..2449c10585 --- /dev/null +++ b/weed/s3api/filer_multipart_list_encoding_test.go @@ -0,0 +1,76 @@ +package s3api + +import ( + "encoding/xml" + "net/url" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// With encoding-type=url the upload keys have to be url-encoded, as ListObjects +// does; otherwise XML 1.0 cannot carry a key with a control character, the +// encoder turns it into U+FFFD, and no client can recover the key to abort the +// upload. +func TestSelectMultipartUploadsEncodingTypeURL(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + input := listUploadsInput("m\x01", "m\x01", "", 1000) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.Len(t, out.Upload, 1) + assert.Equal(t, "m%01p+b%2B.bin", *out.Upload[0].Key) + decoded, err := url.QueryUnescape(*out.Upload[0].Key) + require.NoError(t, err) + assert.Equal(t, "m\x01p b+.bin", decoded) + assert.Equal(t, "m%01", *out.Prefix) + assert.Equal(t, "m%01", *out.KeyMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") + assert.Contains(t, string(body), "m%01p+b%2B.bin") + assert.Contains(t, string(body), "url") +} + +func TestSelectMultipartUploadsEncodingTypeURLNextKeyMarker(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x1fb", 10), + uploadEntry("2", "c", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + out := selectMultipartUploads(input, entries) + + require.True(t, *out.IsTruncated) + assert.Equal(t, "a%1Fb", *out.NextKeyMarker) + // The upload id is opaque, not a key name. + assert.Equal(t, "1", *out.NextUploadIdMarker) + + body, err := xml.Marshal(out) + require.NoError(t, err) + assert.NotContains(t, string(body), "\uFFFD") +} + +// Without encoding-type=url the response is unchanged, and the keys keep their +// raw form for clients that do not ask for encoding. +func TestSelectMultipartUploadsKeepsKeysRawWithoutEncodingType(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "m\x01p b+.bin", 10), + uploadEntry("2", "plain/key.txt", 20), + } + out := selectMultipartUploads(listUploadsInput("", "", "", 1000), entries) + + var got []string + for _, upload := range out.Upload { + got = append(got, *upload.Key) + } + assert.Equal(t, []string{"m\x01p b+.bin", "plain/key.txt"}, got) +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 9a98da6627..945d7862d6 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -290,8 +290,6 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - // TODO handle encodingType - writeSuccessResponseXML(w, r, response) } From 679835e4fc5959e2f054ca09a7d11f50b5b45b86 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:33:51 -0700 Subject: [PATCH 28/70] volume.fsck: run all filer-entry purges through the shared worker pool (#11960) * volume.fsck: queue per-volume orphan deletes through the shared purge pool The worker pool added for -reallyDeleteFilerEntries only covered entries whose volume was missing entirely; the per-volume check that finds filer entries pointing at absent needles still issued one synchronous HTTP DELETE per orphan, which is the bulk of the work the report measured. One pool now serves both purge phases and is drained before the empty- directory sweep. * volume.fsck: keep hard link ids in the filer fid file Orphans queued by the per-volume check must hash into the pool by their hard link id just like the missing-volume path, so two paths sharing an id still serialize on one worker. The fid file now stores the id beside each path and readFilerFileIdFile hands it to the caller. * volume.fsck: cover orphan purge queuing and hard link id round-trip * volume.fsck: fail on a truncated hard link id in the fid file A .fid record cut short inside the hard link id length was emitted with a nil id, so its purge could hash to a different worker than the entries it shares the link with. Read the length strictly like the header and path. * volume.fsck: drain the save consumer before returning a root genFn error The early return left the save goroutine blocked on outputChan while the caller ran finishPurge, so a late queuePurge could send on a closed channel and panic. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * storage: skip EC shard-size encoding tests when disk space is low TestCalculateExpectedShardSizeWithRealEncoding writes a .dat file plus 14 EC shards per subtest (~1.3GB peak for the 512MB case) into the temp dir. Shared CI runners can run low on space while other test packages write to /tmp in parallel, and the encode then dies with ENOSPC mid-write, surfacing as a test failure. Check free space up front and skip the subtest instead. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/shell/command_fs_meta_save.go | 3 + weed/shell/command_volume_fsck.go | 71 +++++++---- weed/shell/command_volume_fsck_test.go | 110 ++++++++++++++++++ .../disk_location_ec_realworld_test.go | 28 ++++- 4 files changed, 183 insertions(+), 29 deletions(-) diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index b936252f62..4034655350 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -323,6 +323,9 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer, firstErr = genErr hasErr.Store(true) }) + close(outputChan) + wg.Wait() + <-saveErrChan return genErr } else { if e.IsDirectory { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 7578c71823..e3c301992d 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -271,19 +271,33 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. if *c.findMissingChunksInFiler { // collect all filer file ids and paths - if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectModifyFromAtNs, collectCutoffFromAtNs); err != nil { - return fmt.Errorf("collectFilerFileIdAndPaths: %w", err) + var queuePurge func(purgeItem) + var finishPurge func() + if *applyPurging || *purgeAbsent { + queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete) } - for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { - // for each volume, check filer file ids - if err = c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, dataNodeId, *applyPurging || *purgeAbsent); err != nil { - return fmt.Errorf("findFilerChunksMissingInVolumeServers: %w", err) + err = func() error { + if err := c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, queuePurge, collectModifyFromAtNs, collectCutoffFromAtNs); err != nil { + return fmt.Errorf("collectFilerFileIdAndPaths: %w", err) } + for dataNodeId, volumeIdToVInfo := range dataNodeVolumeIdToVInfo { + // for each volume, check filer file ids + if err := c.findFilerChunksMissingInVolumeServers(volumeIdToVInfo, dataNodeId, queuePurge); err != nil { + return fmt.Errorf("findFilerChunksMissingInVolumeServers: %w", err) + } + } + return nil + }() + if finishPurge != nil { + finishPurge() + } + if err != nil { + return err } c.purgeEmptyDirectories() } else { // collect all filer file ids - if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, false, 0, 0); err != nil { + if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, false, nil, 0, 0); err != nil { return fmt.Errorf("failed to collect file ids from filer: %w", err) } // If any entry's manifest could not be resolved, our in-use fid set @@ -306,7 +320,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. return nil } -func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, purgeAbsent bool, collectModifyFromAtNs int64, cutoffFromAtNs int64) error { +func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo map[string]map[uint32]VInfo, purgeAbsent bool, queuePurge func(purgeItem), collectModifyFromAtNs int64, cutoffFromAtNs int64) error { if *c.verbose { fmt.Fprintf(c.writer, "checking each file from filer path %s...\n", c.getCollectFilerFilePath()) } @@ -382,12 +396,6 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m }, func(outputChan chan interface{}) error { buffer := make([]byte, readbufferSize) - var queuePurge func(purgeItem) - var finishPurge func() - if purgeAbsent { - queuePurge, finishPurge = startPurgeWorkers(*c.purgeConcurrency, c.httpDelete) - defer finishPurge() - } for item := range outputChan { i := item.(*Item) if f, ok := files[i.vid]; ok { @@ -400,6 +408,13 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m if _, err := f.Write([]byte(i.path)); err != nil { return err } + util.Uint32toBytes(buffer[:4], uint32(len(i.hardLinkId))) + if _, err := f.Write(buffer[:4]); err != nil { + return err + } + if _, err := f.Write(i.hardLinkId); err != nil { + return err + } } else if *c.findMissingChunksInFiler { // check if the volume matches the filter if len(c.volumeIds) > 0 { @@ -423,10 +438,10 @@ func (c *commandVolumeFsck) collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo m }) } -func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, dataNodeId string, applyPurging bool) error { +func (c *commandVolumeFsck) findFilerChunksMissingInVolumeServers(volumeIdToVInfo map[uint32]VInfo, dataNodeId string, queuePurge func(purgeItem)) error { for volumeId, vinfo := range volumeIdToVInfo { - checkErr := c.oneVolumeFileIdsCheckOneVolume(dataNodeId, volumeId, applyPurging) + checkErr := c.oneVolumeFileIdsCheckOneVolume(dataNodeId, volumeId, queuePurge) if checkErr != nil { return fmt.Errorf("failed to collect file ids from volume %d on %s: %v", volumeId, vinfo.server, checkErr) } @@ -650,7 +665,7 @@ func (lw *lockedWriter) Write(p []byte) (int, error) { return lw.w.Write(p) } -func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath)) error { +func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleId types.NeedleId, itemPath util.FullPath, hardLinkId []byte)) error { fp, err := os.Open(getFilerFileIdFile(c.tempFolder, volumeId)) if err != nil { return err @@ -685,13 +700,23 @@ func (c *commandVolumeFsck) readFilerFileIdFile(volumeId uint32, fn func(needleI return fmt.Errorf("read fid path size mismatch for volume %d,%x%08x: got %d want %d", volumeId, item.fileKey, item.cookie, n, pathSize) } item.path = util.FullPath(pathBytes) + item.hardLinkId = nil + if _, err := io.ReadFull(br, buffer[:4]); err != nil { + return fmt.Errorf("read fid hard link id size for volume %d,%x%08x: %w", volumeId, item.fileKey, item.cookie, err) + } + if hardLinkIdSize := util.BytesToUint32(buffer[:4]); hardLinkIdSize > 0 { + item.hardLinkId = make([]byte, int(hardLinkIdSize)) + if _, err := io.ReadFull(br, item.hardLinkId); err != nil { + return fmt.Errorf("read fid hard link id for volume %d,%x%08x: %w", volumeId, item.fileKey, item.cookie, err) + } + } needleId := types.NeedleId(item.fileKey) - fn(needleId, item.path) + fn(needleId, item.path, item.hardLinkId) } return nil } -func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, volumeId uint32, applyPurging bool) (err error) { +func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, volumeId uint32, queuePurge func(purgeItem)) (err error) { if *c.verbose { fmt.Fprintf(c.writer, "find missing file chunks in dataNodeId %s volume %d ...\n", dataNodeId, volumeId) } @@ -702,11 +727,11 @@ func (c *commandVolumeFsck) oneVolumeFileIdsCheckOneVolume(dataNodeId string, vo if err = db.LoadFromIdx(getVolumeFileIdFile(c.tempFolder, dataNodeId, volumeId)); err != nil { return } - if err = c.readFilerFileIdFile(volumeId, func(needleId types.NeedleId, itemPath util.FullPath) { + if err = c.readFilerFileIdFile(volumeId, func(needleId types.NeedleId, itemPath util.FullPath, hardLinkId []byte) { if _, found := db.Get(needleId); !found { fmt.Fprintf(c.writer, "%s\n", itemPath) - if applyPurging { - c.httpDelete(itemPath) + if queuePurge != nil { + queuePurge(purgeItem{path: itemPath, hardLinkId: hardLinkId}) } } }); err != nil { @@ -889,7 +914,7 @@ func (c *commandVolumeFsck) oneVolumeFileIdsSubtractFilerFileIds(dataNodeId stri return } - if err = c.readFilerFileIdFile(volumeId, func(filerNeedleId types.NeedleId, itemPath util.FullPath) { + if err = c.readFilerFileIdFile(volumeId, func(filerNeedleId types.NeedleId, itemPath util.FullPath, _ []byte) { inUseCount++ if *c.verifyNeedle && !vinfo.isEcVolume { if needleValue, ok := volumeFileIdDb.Get(filerNeedleId); ok && !needleValue.Size.IsDeleted() { diff --git a/weed/shell/command_volume_fsck_test.go b/weed/shell/command_volume_fsck_test.go index 50bdd0f8fa..7804b85224 100644 --- a/weed/shell/command_volume_fsck_test.go +++ b/weed/shell/command_volume_fsck_test.go @@ -1,12 +1,14 @@ package shell import ( + "bytes" "context" "fmt" "io" "net" "net/http" "net/http/httptest" + "os" "sort" "strings" "sync/atomic" @@ -16,6 +18,8 @@ import ( "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/storage/needle_map" + "github.com/seaweedfs/seaweedfs/weed/storage/types" "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -310,3 +314,109 @@ func TestVolumeFsckPurgeHardLinksSerialized(t *testing.T) { t.Fatalf("processed %d deletes, want 8", got) } } + +// Orphans found by the per-volume check go through the same purge queue as +// the missing-volume path, and the fid file keeps each entry's hard link id +// so same-id deletes still land on one worker. +func TestVolumeFsckCheckOneVolumeQueuesOrphansWithHardLinkId(t *testing.T) { + tempDir := t.TempDir() + var verbose bool + var buf bytes.Buffer + c := &commandVolumeFsck{tempFolder: tempDir, verbose: &verbose, writer: &buf} + + volumeId := uint32(7) + db := needle_map.NewMemDb() + if err := db.Set(types.NeedleId(11), types.ToOffset(16), types.Size(1)); err != nil { + t.Fatalf("seed volume idx: %v", err) + } + if err := db.SaveToIdx(getVolumeFileIdFile(tempDir, "dn1", volumeId)); err != nil { + t.Fatalf("write volume idx: %v", err) + } + db.Close() + + f, err := os.Create(getFilerFileIdFile(tempDir, volumeId)) + if err != nil { + t.Fatalf("create fid file: %v", err) + } + writeFid := func(fileKey uint64, cookie uint32, p string, hardLinkId []byte) { + buffer := make([]byte, readbufferSize) + util.Uint64toBytes(buffer, fileKey) + util.Uint32toBytes(buffer[8:], cookie) + util.Uint32toBytes(buffer[12:], uint32(len(p))) + if _, err := f.Write(buffer); err != nil { + t.Fatalf("write fid header: %v", err) + } + if _, err := f.Write([]byte(p)); err != nil { + t.Fatalf("write fid path: %v", err) + } + util.Uint32toBytes(buffer[:4], uint32(len(hardLinkId))) + if _, err := f.Write(buffer[:4]); err != nil { + t.Fatalf("write fid hard link id size: %v", err) + } + if _, err := f.Write(hardLinkId); err != nil { + t.Fatalf("write fid hard link id: %v", err) + } + } + writeFid(11, 1, "/keep/a", nil) + writeFid(22, 2, "/orphan/b", []byte{1, 2, 3}) + writeFid(33, 3, "/orphan/c", nil) + if err := f.Close(); err != nil { + t.Fatalf("close fid file: %v", err) + } + + var queued []purgeItem + if err := c.oneVolumeFileIdsCheckOneVolume("dn1", volumeId, func(it purgeItem) { + queued = append(queued, it) + }); err != nil { + t.Fatalf("check volume: %v", err) + } + if len(queued) != 2 { + t.Fatalf("queued %d purges, want 2 (the two orphans): %v", len(queued), queued) + } + if queued[0].path != "/orphan/b" || queued[1].path != "/orphan/c" { + t.Fatalf("queued paths %v, want /orphan/b then /orphan/c", queued) + } + if !bytes.Equal(queued[0].hardLinkId, []byte{1, 2, 3}) { + t.Fatalf("queued hardLinkId %v, want [1 2 3]", queued[0].hardLinkId) + } + if queued[1].hardLinkId != nil { + t.Fatalf("queued hardLinkId %v, want nil", queued[1].hardLinkId) + } +} + +// A .fid file cut short inside the hard link id field is corrupt input, not a +// record without a link id: the reader must fail instead of emitting the entry +// with a nil id. +func TestReadFilerFileIdFileRejectsTruncatedHardLinkId(t *testing.T) { + tempDir := t.TempDir() + var verbose bool + c := &commandVolumeFsck{tempFolder: tempDir, verbose: &verbose, writer: io.Discard} + + f, err := os.Create(getFilerFileIdFile(tempDir, 7)) + if err != nil { + t.Fatalf("create fid file: %v", err) + } + buffer := make([]byte, readbufferSize) + util.Uint64toBytes(buffer, 11) + util.Uint32toBytes(buffer[8:], 1) + util.Uint32toBytes(buffer[12:], uint32(len("/orphan/b"))) + if _, err := f.Write(buffer); err != nil { + t.Fatalf("write fid header: %v", err) + } + if _, err := f.Write([]byte("/orphan/b")); err != nil { + t.Fatalf("write fid path: %v", err) + } + if _, err := f.Write([]byte{0, 0}); err != nil { + t.Fatalf("write partial hard link id size: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close fid file: %v", err) + } + + err = c.readFilerFileIdFile(7, func(types.NeedleId, util.FullPath, []byte) { + t.Fatal("callback fired for a truncated record") + }) + if err == nil { + t.Fatal("expected an error for a truncated hard link id size") + } +} diff --git a/weed/storage/disk_location_ec_realworld_test.go b/weed/storage/disk_location_ec_realworld_test.go index 42ce29bd78..f54682a47d 100644 --- a/weed/storage/disk_location_ec_realworld_test.go +++ b/weed/storage/disk_location_ec_realworld_test.go @@ -5,9 +5,24 @@ import ( "path/filepath" "testing" + "github.com/seaweedfs/seaweedfs/weed/stats" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" ) +// requireDiskSpaceForEncoding skips the subtest when dir's filesystem cannot +// hold the .dat file plus the shards EC encoding writes for it, so ENOSPC on +// space-constrained runners doesn't surface as a test failure. +func requireDiskSpaceForEncoding(t *testing.T, dir string, datFileSize int64, expectedShardSize int64) { + t.Helper() + needBytes := uint64(datFileSize) + + uint64(expectedShardSize)*erasure_coding.TotalShardsCount + + 16*erasure_coding.ErasureCodingSmallBlockSize + if free := stats.NewDiskStatus(dir).Free; free < needBytes { + t.Skipf("insufficient free space in %s: have %d bytes, need ~%d to encode a %d-byte volume", + dir, free, needBytes, datFileSize) + } +} + // TestCalculateExpectedShardSizeWithRealEncoding validates our shard size calculation // by actually running EC encoding on real files and comparing the results func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { @@ -52,6 +67,10 @@ func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + // Calculate expected shard size using our function + expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) + requireDiskSpaceForEncoding(t, tempDir, tt.datFileSize, expectedShardSize) + // Create a test .dat file with the specified size baseFileName := filepath.Join(tempDir, "test_volume") datFileName := baseFileName + ".dat" @@ -82,9 +101,6 @@ func TestCalculateExpectedShardSizeWithRealEncoding(t *testing.T) { } datFile.Close() - // Calculate expected shard size using our function - expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) - // Run actual EC encoding _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()) if err != nil { @@ -146,6 +162,9 @@ func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) + requireDiskSpaceForEncoding(t, tempDir, tt.datFileSize, expectedShardSize) + baseFileName := filepath.Join(tempDir, tt.name) datFileName := baseFileName + ".dat" @@ -163,9 +182,6 @@ func TestCalculateExpectedShardSizeEdgeCases(t *testing.T) { datFile.Write(data) datFile.Close() - // Calculate expected - expectedShardSize := calculateExpectedShardSize(tt.datFileSize, erasure_coding.DataShardsCount) - // Run actual EC encoding _, err = erasure_coding.WriteEcFiles(baseFileName, erasure_coding.BackgroundECContext()) if err != nil { From 85f2f9d2d4433b27e7d56dfa73eb6e8e2f005026 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:37:36 -0700 Subject: [PATCH 29/70] ci: make docker registry and compose steps idempotent on self-hosted runners (#12012) The registry container is started with --restart=always but never removed during teardown, so once a machine runs the job, every later run on that machine fails with a container name conflict. Reuse the running container when it exists and recreate it only when missing or broken. Some self-hosted runners (e.g. seaweedfs-server03) also lack the Docker Compose plugin, so install it via docker/setup-compose-action, which is a no-op where compose is already available. --- .github/workflows/pjdfstest.yml | 9 ++++++++- .github/workflows/samba-integration.yml | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pjdfstest.yml b/.github/workflows/pjdfstest.yml index 878f787f4f..a88d19d48a 100644 --- a/.github/workflows/pjdfstest.yml +++ b/.github/workflows/pjdfstest.yml @@ -41,13 +41,20 @@ jobs: cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry - run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + run: | + if ! docker start registry 2>/dev/null; then + docker rm -f registry >/dev/null 2>&1 || true + docker run -d --restart=always -p 5000:5000 --name registry registry:2 + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver-opts: network=host + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + - name: Build weed race binary run: | cd docker diff --git a/.github/workflows/samba-integration.yml b/.github/workflows/samba-integration.yml index f37020090a..94b9d670d3 100644 --- a/.github/workflows/samba-integration.yml +++ b/.github/workflows/samba-integration.yml @@ -43,13 +43,20 @@ jobs: cache: ${{ runner.environment == 'github-hosted' }} - name: Start local Docker registry - run: docker run -d --restart=always -p 5000:5000 --name registry registry:2 + run: | + if ! docker start registry 2>/dev/null; then + docker rm -f registry >/dev/null 2>&1 || true + docker run -d --restart=always -p 5000:5000 --name registry registry:2 + fi - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 with: driver-opts: network=host + - name: Set up Docker Compose + uses: docker/setup-compose-action@v2 + - name: Build weed race binary run: | cd docker From 0d4e11a6e5b0c7fb855388936d6bf79ad5001a2a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:45:09 -0700 Subject: [PATCH 30/70] filer: bootstrap a timestamp-less peer entry onto a missing local path (#11975) A zero local commit time conflated 'no local state' with 'state at the epoch', so a missing entry was skipped whenever the sender reported no commit time and the entry mtime was at or before the epoch. Only hold the copy off when a local entry actually exists. --- weed/filer/meta_aggregator.go | 17 ++++++++++- weed/filer/meta_aggregator_replay_test.go | 37 +++++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/weed/filer/meta_aggregator.go b/weed/filer/meta_aggregator.go index c5d6521c89..310874a2e2 100644 --- a/weed/filer/meta_aggregator.go +++ b/weed/filer/meta_aggregator.go @@ -788,7 +788,22 @@ func (f *Filer) applyTraversedEntry(ctx context.Context, entry *Entry, peerTsNs return fmt.Errorf("check local state for %s: %w", entry.FullPath, err) } if localTsNs >= peerTsNs { - return nil + if localTsNs != 0 { + return nil + } + // Zero timestamps carry no order: keep the path only when an entry + // exists locally — a missing one takes the peer's copy, or an + // epoch-dated entry from a peer reporting no commit time is dropped. + if f.Store == nil { + return nil + } + local, findErr := f.Store.FindEntry(ctx, entry.FullPath) + if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) { + return fmt.Errorf("check local state for %s: %w", entry.FullPath, findErr) + } + if local != nil { + return nil + } } if insertErr := f.Store.InsertEntry(ctx, entry); insertErr != nil { // Entry already exists (root dir, or partial previous bootstrap); diff --git a/weed/filer/meta_aggregator_replay_test.go b/weed/filer/meta_aggregator_replay_test.go index 3e700a47ec..efbc9968f2 100644 --- a/weed/filer/meta_aggregator_replay_test.go +++ b/weed/filer/meta_aggregator_replay_test.go @@ -575,6 +575,43 @@ func TestBootstrapUpdatesExistingByCommitTime(t *testing.T) { } } +// A missing local entry has no commit time to order against: the peer copy +// must apply even when both sides are timestamp-less — an epoch-dated entry +// from a peer that predates EntryTsNs is still bootstrapped, while an +// existing zero-mtime local entry keeps its place. +func TestBootstrapCopiesMissingEntryWithoutCommitTime(t *testing.T) { + store := newStubFilerStore() + f := newReplayTestFiler(store) + missing := util.NewFullPath("/dir", "missing") + existing := util.NewFullPath("/dir", "existing") + + store.entries[string(existing)] = &Entry{ + FullPath: existing, + Attr: Attr{Mtime: time.Unix(0, 0)}, + Content: []byte("zero-mtime local entry"), + } + for _, path := range []util.FullPath{missing, existing} { + entry := &Entry{ + FullPath: path, + Attr: Attr{Mtime: time.Unix(0, 0)}, + Content: []byte("epoch-dated peer entry"), + } + if err := f.applyTraversedEntry(context.Background(), entry, 0); err != nil { + t.Fatalf("applyTraversedEntry %s: %v", path, err) + } + } + got, err := store.FindEntry(context.Background(), missing) + if err != nil { + t.Fatalf("missing local entry must take the peer copy: %v", err) + } + if string(got.Content) != "epoch-dated peer entry" { + t.Fatalf("content = %q, want peer entry applied", got.Content) + } + if got, _ := store.FindEntry(context.Background(), existing); string(got.Content) != "zero-mtime local entry" { + t.Fatalf("content = %q, existing local entry must be kept", got.Content) + } +} + // A delete whose commit record failed to persist still blocks a stale // replayed create: the pending record stands in until the retry lands it. func TestPendingDeleteRecordBlocksStaleReplay(t *testing.T) { From 462e4445e2f3a2413ddfd9463b03740134878d20 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:46:26 -0700 Subject: [PATCH 31/70] test: skip catalog client images that cannot execute on the host CPU (#12014) New upstream releases build for x86-64-v3 (AVX2) - ClickHouse since 26.5, plus the DuckDB iceberg extension and pylance wheels - so on a runner whose CPU lacks AVX2 the container dies on SIGILL before the test reaches any SeaweedFS code. Probe each third-party image first and skip on the illegal-instruction exit instead of failing; other errors still surface normally. --- .../clickhouse_catalog_test.go | 5 +++ .../lifecycle/iceberg_lifecycle_test.go | 10 ++++++ .../lifecycle/lance_lifecycle_test.go | 3 ++ test/testutil/helpers.go | 31 +++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go index 098fae43a5..b85629fed4 100644 --- a/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go +++ b/test/s3tables/catalog_clickhouse/clickhouse_catalog_test.go @@ -871,6 +871,11 @@ func requireClickHouseRuntime(t *testing.T) { if !hasDocker() { t.Skip("Docker not available, skipping ClickHouse integration test") } + // ClickHouse builds for x86-64-v3 (AVX2) since 26.5; on a CPU without it + // the binary dies on SIGILL inside the entrypoint before the server can + // come up. + testutil.RequireImageRunnable(t, + "--entrypoint", "clickhouse", clickhouseImage(), "local", "--query", "SELECT 1") } // hasDocker reports whether `docker version` can run, which we treat as a diff --git a/test/s3tables/lifecycle/iceberg_lifecycle_test.go b/test/s3tables/lifecycle/iceberg_lifecycle_test.go index 2c34d7b5ce..0eae42a575 100644 --- a/test/s3tables/lifecycle/iceberg_lifecycle_test.go +++ b/test/s3tables/lifecycle/iceberg_lifecycle_test.go @@ -70,6 +70,13 @@ func TestIcebergTableLifecycle(t *testing.T) { func testIcebergLifecycleWithDuckDB(t *testing.T) { env := shared + + // The write phase's first act is INSTALLing the iceberg extension, whose + // binary can itself target a newer x86-64 baseline than the host CPU has. + testutil.RequireImageRunnable(t, + "--entrypoint", "duckdb", duckdbImage, + "-c", "INSTALL iceberg; LOAD iceberg; SELECT 1;") + bucket := "lifecycle-duckdb-" + randomSuffix() namespace, table := "sales", "events" env.createTableBucket(t, bucket, "ICEBERG") @@ -109,6 +116,9 @@ func testIcebergLifecycleWithPyIceberg(t *testing.T) { namespace, table := "sales", "events" env.createTableBucket(t, bucket, "ICEBERG") buildClientImage(t, pyicebergImage, "Dockerfile.pyiceberg") + testutil.RequireImageRunnable(t, + "--entrypoint", "python3", pyicebergImage, + "-c", "import pyarrow; from pyiceberg.catalog import load_catalog") before := env.pyiceberg(t, "write", bucket, namespace, table) assertSeeded(t, before) diff --git a/test/s3tables/lifecycle/lance_lifecycle_test.go b/test/s3tables/lifecycle/lance_lifecycle_test.go index ea076eef55..31b5b6c173 100644 --- a/test/s3tables/lifecycle/lance_lifecycle_test.go +++ b/test/s3tables/lifecycle/lance_lifecycle_test.go @@ -35,6 +35,9 @@ func TestLanceTableLifecycle(t *testing.T) { namespace, table := "ml", "events" env.createTableBucket(t, bucket, "LANCE") buildClientImage(t, lanceImage, "Dockerfile.lance") + testutil.RequireImageRunnable(t, + "--entrypoint", "python3", lanceImage, + "-c", "import lance; import lance_namespace; import pyarrow") before := env.lance(t, "write", bucket, namespace, table) assertSeeded(t, before) diff --git a/test/testutil/helpers.go b/test/testutil/helpers.go index 268b35b29f..8d2d6b2806 100644 --- a/test/testutil/helpers.go +++ b/test/testutil/helpers.go @@ -2,12 +2,15 @@ package testutil import ( "context" + "errors" "fmt" "net" "net/http" "os" "os/exec" "path/filepath" + "strings" + "testing" "time" ) @@ -20,6 +23,34 @@ func HasDocker() bool { return cmd.Run() == nil } +// RequireImageRunnable runs a throwaway container with the given `docker run` +// arguments and skips the test when its process dies on SIGILL. Upstream +// images increasingly target x86-64-v3 (AVX2) - ClickHouse since 26.5 - so a +// host CPU without those instructions cannot run them at all, and the failure +// says nothing about the code under test. Any other outcome, whether a clean +// run, an ordinary non-zero exit, or a pull failure, is left for the test +// itself to report: the probe can only skip on the one failure the code +// cannot cause. +func RequireImageRunnable(t *testing.T, args ...string) { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", append([]string{"run", "--rm"}, args...)...) + out, err := cmd.CombinedOutput() + if err == nil { + return + } + var exitErr *exec.ExitError + // 132 = 128 + SIGILL: docker propagates the container process's death. + if errors.As(err, &exitErr) && + (exitErr.ExitCode() == 132 || + strings.Contains(string(out), "Illegal instruction")) { + t.Skipf("image cannot execute on this CPU (illegal instruction): docker run %s\n%s", + strings.Join(args, " "), out) + } +} + func FindBindIP() string { addrs, err := net.InterfaceAddrs() if err != nil { From ce4c1fe31e0c565e053e032bc5f0db9814ae9c23 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:49:04 -0700 Subject: [PATCH 32/70] ci: install fuse3 when runners lack fusermount3 (#12013) Self-hosted runners do not ship fuse3, so the setuid repair step failed with 'no fusermount3 in PATH' before the FUSE jobs could run. Install the package instead of failing outright. --- .github/actions/fix-fusermount-setuid/action.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/actions/fix-fusermount-setuid/action.yml b/.github/actions/fix-fusermount-setuid/action.yml index 7aad5bf482..5b0428c713 100644 --- a/.github/actions/fix-fusermount-setuid/action.yml +++ b/.github/actions/fix-fusermount-setuid/action.yml @@ -23,8 +23,14 @@ runs: bin=$(command -v fusermount3 || true) if [ -z "$bin" ]; then - echo "no fusermount3 in PATH" >&2 - exit 1 + # Self-hosted runners are not guaranteed to carry fuse3. + sudo apt-get update + sudo apt-get install -y fuse3 + bin=$(command -v fusermount3 || true) + if [ -z "$bin" ]; then + echo "no fusermount3 in PATH after installing fuse3" >&2 + exit 1 + fi fi if can_mount "$bin"; then ls -l "$bin" From 617599859ba0579befdc74b23ed3f707e65515ee Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 18:50:27 -0700 Subject: [PATCH 33/70] docker: build RocksDB v11.1.2 for grocksdb v1.11.1 (#12002) The rocksdb release image fails to compile: grocksdb v1.11.1 calls C.rocksdb_block_based_options_set_block_align, C.rocksdb_block_based_options_set_index_block_search_type, and C.rocksdb_block_based_options_set_separate_key_value_in_data_block, all of which only exist in RocksDB v11.x headers. grocksdb v1.11.1's own build script pins 11.1.2, so move every default from v10.10.1 to v11.1.2 to match. Generated with [Devin](https://devin.ai) Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/container_release_unified.yml | 4 ++-- .github/workflows/container_rocksdb_version.yml | 4 ++-- docker/Dockerfile.rocksdb_dev_env | 2 +- docker/Dockerfile.rocksdb_large | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 6ac4ce06db..3080595eb4 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -25,7 +25,7 @@ on: rocksdb_version: description: 'RocksDB git tag to use when variant=rocksdb' required: false - default: 'v10.10.1' + default: 'v11.1.2' permissions: contents: read @@ -282,7 +282,7 @@ jobs: ${{ matrix.build_args }} BUILDKIT_INLINE_CACHE=1 BRANCH=${{ github.sha }} - ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v10.10.1') || '' }} + ${{ matrix.variant == 'rocksdb' && format('ROCKSDB_VERSION={0}', github.event.inputs.rocksdb_version || 'v11.1.2') || '' }} - name: Export digest if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/container_rocksdb_version.yml b/.github/workflows/container_rocksdb_version.yml index f06eb500f4..0d3b5f2b8f 100644 --- a/.github/workflows/container_rocksdb_version.yml +++ b/.github/workflows/container_rocksdb_version.yml @@ -4,9 +4,9 @@ on: workflow_dispatch: inputs: rocksdb_version: - description: 'RocksDB git tag or branch to build (e.g. v10.10.1)' + description: 'RocksDB git tag or branch to build (e.g. v11.1.2)' required: true - default: 'v10.10.1' + default: 'v11.1.2' seaweedfs_ref: description: 'SeaweedFS git tag, branch, or commit to build' required: true diff --git a/docker/Dockerfile.rocksdb_dev_env b/docker/Dockerfile.rocksdb_dev_env index 95154c3737..2b6b0431b1 100644 --- a/docker/Dockerfile.rocksdb_dev_env +++ b/docker/Dockerfile.rocksdb_dev_env @@ -3,7 +3,7 @@ FROM golang:1.26 AS builder RUN apt-get update RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev -ARG ROCKSDB_VERSION=v10.10.1 +ARG ROCKSDB_VERSION=v11.1.2 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} # build RocksDB diff --git a/docker/Dockerfile.rocksdb_large b/docker/Dockerfile.rocksdb_large index d902c544e9..757af7e64d 100644 --- a/docker/Dockerfile.rocksdb_large +++ b/docker/Dockerfile.rocksdb_large @@ -3,7 +3,7 @@ FROM golang:1.26 AS builder RUN apt-get update RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev -ARG ROCKSDB_VERSION=v10.10.1 +ARG ROCKSDB_VERSION=v11.1.2 ENV ROCKSDB_VERSION=${ROCKSDB_VERSION} # build RocksDB From cd562b8514226d7377a2c0efbd33001867ff2153 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2026 02:56:41 +0000 Subject: [PATCH 34/70] docs: regenerate star history chart --- note/star_history.svg | 2174 ++++++++++++++++++++--------------------- 1 file changed, 1080 insertions(+), 1094 deletions(-) diff --git a/note/star_history.svg b/note/star_history.svg index ad8995f239..064e5b443b 100644 --- a/note/star_history.svg +++ b/note/star_history.svg @@ -6,7 +6,7 @@ - 2026-10-23T02:28:24.716056 + 2026-10-24T02:56:41.212930 image/svg+xml @@ -40,23 +40,23 @@ z - + - - + - + - + - + - + - + - + - + - + - + - + @@ -301,18 +301,18 @@ L 353.430232 26.215385 - + - + - + @@ -325,18 +325,18 @@ L 449.355659 26.215385 - + - + - + @@ -349,18 +349,18 @@ L 545.149861 26.215385 - + - + - + @@ -484,41 +484,41 @@ z - + - - + - + - + - + - + - + - + - + @@ -577,18 +577,18 @@ L 709.2 267.748071 - + - + - + @@ -599,18 +599,18 @@ L 709.2 223.798916 - + - + - + @@ -621,18 +621,18 @@ L 709.2 179.849762 - + - + - + @@ -643,18 +643,18 @@ L 709.2 135.900608 - + - + - + - + - + - + @@ -813,1023 +813,1009 @@ z +" clip-path="url(#p16b1f4eee4)" style="fill: none; stroke: #0969da; stroke-width: 1.6; stroke-linecap: square"/> + From 6e8a2cdb149b93623e90631f2800523738866726 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 20:02:55 -0700 Subject: [PATCH 35/70] filer.backup: process metadata events through the concurrent processor (#11951) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * filer.backup: process metadata events through the concurrent processor Each event was handled inline on the subscription loop, so one fetch+write at a time capped the stream at a few entries per second and a busy source kept building lag. Route events through the same MetadataProcessor filer.sync uses: it fans jobs out behind -concurrency while serializing same-path and subtree-barrier conflicts, and the checkpoint now advances on the processor's contiguous watermark via GetResumeTsNs, with Resubscribe replaying anything a failure pins. -disableErrorRetry keeps its log-and-skip meaning by returning nil after logging, since the processor would otherwise pin the checkpoint forever. * filer.backup: drain in-flight jobs before the stream retry loop reconnects FollowMetadata returns as soon as the stream ends while admitted jobs keep writing; the next doFilerBackup builds a fresh processor that cannot order its replayed writes against them, so an older upload could finish last and clobber a newer file. * filer.backup: checkpoint the completed watermark on a ticker and at stream end The offset callback only fires while events arrive, so work draining on a quiet stream stayed uncheckpointed until the next event and a restart recopied it. Persist the processed watermark every 3s when it advances, and once more after the post-stream drain. * filer.backup: bound the checkpoint wait before reconnecting setOffset has no context, so a stalled KvPut would hold persistDone and block the retry loop forever. The persist goroutine now performs the final write itself on stopPersist, and the caller waits at most 30s. * filer.backup: carry the processed position across retry attempts A follow attempt that ended with a pinned failure recomputed startFrom on retry, so a -timeAgo window could move past the failed event and skip it. doFilerBackup now returns the failure pin (or processed watermark) and the retry loop clamps startFrom to it. * filer.backup: resume retries at the previous attempt's pinned position The retry loop threaded resumeTsNs through but never consulted it, so a -timeAgo run recomputed startFrom as now-minus-the-window on every attempt — a failed event older than the window was skipped on reconnect. Use the pinned position when one exists; -timeAgo and the persisted checkpoint only choose the first starting point. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.backup: resume below the failure pin so the failed event replays The metadata reader treats the resume position as exclusive — events at or below it are skipped — so resuming at the failed event's own timestamp would skip it. Back one tick off the pin. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.backup: label the progress line as the received-event rate The counter increments when a job is admitted, not when its copy finishes, so the old wording showed backup throughput during a burst of slow copies while the processed watermark had not moved. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * replication: bound the offset checkpoint RPC setOffset called KvPut on the background context, so a stalled write could pin a caller's drain or persistence loop indefinitely. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.backup: keep the resume pin when filer-source setup fails A retried attempt carries resumeTsNs so the failed event replays from the same position, but filer-source and SSE initialization failures returned 0, so the next attempt recomputed -timeAgo and could move the window past the pinned event. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * replication: serialize checkpoint writes per key A checkpoint write can outlive its caller's wait and land beside the next subscription attempt's writes, moving saved progress backward. Serialize writes per key inside setOffset and drop any value older than one already written from this process. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.backup: gofmt * replication: serialize checkpoint writes per filer and key A single process-wide lock made a stalled checkpoint RPC block every stream's checkpoint, including the opposite direction of a bidirectional filer.sync. Keeping the last-written watermark per persisted key also let one stream's offset shadow another stream checkpointing the same key to a different target filer. Lock and track progress per (filer, key). * replication: keep the resume position behind a cleared failure pin A redelivery that clears failedTs lets the processed watermark advance past the earliest failure, but events dropped while the processor was stopped — possibly sharing the failure's timestamp — have to replay. Latched earliestFailedTsNs floors the resume position one tick below that failure so reconnects still cover them. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * replication: persist floored checkpoints through the write ordering setOffset drops a value older than the last written, which also swallows a deliberate regression: the resume floor, an operator-seeded offset, or a snapshot high-water mark. allowRegress marks those callers so the checkpoint can move backward when a failure forces replay. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * replication: order checkpoint writes by issuance, not value The value-based stale guard could never fire — every caller passed allowRegress because floors and seeds are deliberate regressions — and ordering by value cannot tell a stale high watermark apart from real progress: a write that outlives its attempt could still land after a floored resume and skip events owed a replay. Order by issuance instead: a write issued before the last one to land under the key is dropped, so late writes from a superseded attempt can no longer move the checkpoint in either direction while deliberate regressions still apply. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * replication: keep the resume position one tick below the frontier * remote.sync: resume and persist through the floored watermark * test: concurrent metadata jobs fan out up to the limit Distinct-path events must all be in flight before any finishes — a serial loop could only ever have one running — while the limit still bounds the pool. Pins the property behind the backup-throughput fix. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/filer_backup.go | 119 ++++++++++++-- weed/command/filer_remote_gateway_buckets.go | 4 +- weed/command/filer_remote_sync_dir.go | 4 +- weed/command/filer_sync.go | 79 +++++++-- weed/command/filer_sync_jobs.go | 50 ++++++ weed/command/filer_sync_jobs_test.go | 160 +++++++++++++++++++ weed/command/filer_sync_offset_test.go | 53 +++++- 7 files changed, 443 insertions(+), 26 deletions(-) diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 96b363cf55..060c5fc837 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -42,6 +42,7 @@ type FilerBackupOptions struct { timeAgo *time.Duration retentionDays *int initialSnapshot *bool + concurrency *int } var ( @@ -65,6 +66,7 @@ func init() { filerBackupOptions.disableErrorRetry = cmdFilerBackup.Flag.Bool("disableErrorRetry", false, "disables errors retry, only logs will print") filerBackupOptions.ignore404Error = cmdFilerBackup.Flag.Bool("ignore404Error", true, "ignore 404 errors from filer") filerBackupOptions.initialSnapshot = cmdFilerBackup.Flag.Bool("initialSnapshot", false, "before subscribing to metadata updates, walk the live filer tree under -filerPath and seed the destination, then subscribe from the walk-start timestamp so concurrent changes are still captured. Runs on every start when -timeAgo is 0 and overwrites the saved checkpoint, so it also re-seeds a reinitialized destination. Remove it once the backup is caught up, otherwise it re-walks the whole tree on each restart.") + filerBackupOptions.concurrency = cmdFilerBackup.Flag.Int("concurrency", DefaultConcurrencyLimit, "the maximum number of files that will be backed up concurrently.") } var cmdFilerBackup = &Command{ @@ -105,10 +107,12 @@ func runFilerBackup(cmd *Command, args []string) bool { clientId := util.RandomInt32() var clientEpoch int32 + var resumeTsNs int64 for { clientEpoch++ - err := doFilerBackup(grpcDialOption, &filerBackupOptions, reExcludeFileName, excludeFileNames, excludePathPatterns, clientId, clientEpoch) + var err error + resumeTsNs, err = doFilerBackup(grpcDialOption, &filerBackupOptions, reExcludeFileName, excludeFileNames, excludePathPatterns, clientId, clientEpoch, resumeTsNs) if err != nil { glog.Errorf("backup from %s: %v", *filerBackupOptions.filer, err) time.Sleep(1747 * time.Millisecond) @@ -120,12 +124,12 @@ const ( BackupKeyPrefix = "backup." ) -func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, clientId int32, clientEpoch int32) error { +func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOptions, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, clientId int32, clientEpoch int32, resumeTsNs int64) (int64, error) { // find data sink dataSink := findSink(util.GetViper()) if dataSink == nil { - return fmt.Errorf("no data sink configured in replication.toml") + return 0, fmt.Errorf("no data sink configured in replication.toml") } sourceFiler := pb.ServerAddress(*backupOption.filer) @@ -139,7 +143,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti startFrom := time.Unix(0, 0) sinkId, legacySinkId := backupCheckpointIds(sourcePath, dataSink) runSnapshot := *backupOption.initialSnapshot && timeAgo == 0 - if timeAgo == 0 { + if resumeTsNs > 0 { + // The previous attempt pinned where it stopped — a failed event's + // timestamp or the processed watermark — which can sit behind both the + // persisted checkpoint and a recomputed -timeAgo window. + startFrom = time.Unix(0, resumeTsNs) + glog.V(0).Infof("resuming previous attempt at %v", startFrom) + } else if timeAgo == 0 { if runSnapshot { // snapshot below sets the start point; no checkpoint read needed glog.V(0).Infof("initialSnapshot requested — walking live tree before subscribing") @@ -166,11 +176,11 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler); err != nil { - return fmt.Errorf("filersource initialization failed: %v", err) + return resumeTsNs, fmt.Errorf("filersource initialization failed: %v", err) } if err := repl_util.InitializeSSEForReplication(filerSource); err != nil { - return fmt.Errorf("SSE initialization failed: %v", err) + return resumeTsNs, fmt.Errorf("SSE initialization failed: %v", err) } dataSink.SetSourceFiler(filerSource) @@ -181,14 +191,14 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti if runSnapshot { snapshotTsNs, err := runInitialSnapshot(sourceFiler.ToGrpcAddress(), filerSource, sourcePath, targetPath, excludePaths, reExcludeFileName, excludeFileNames, excludePathPatterns, dataSink, *backupOption.ignore404Error) if err != nil { - return fmt.Errorf("initial snapshot: %w", err) + return 0, fmt.Errorf("initial snapshot: %w", err) } // The walk can take hours on large trees; retry the tiny KV write a // handful of times before giving up so a flaky filer KV doesn't force // the whole walk to repeat on the next retry loop iteration. if err := persistSnapshotOffset(grpcDialOption, sourceFiler, sinkId, snapshotTsNs); err != nil { glog.Errorf("initialSnapshot: FAILED to persist offset %d for sinkId %d after retries: %v — the next retry will redo the full walk", snapshotTsNs, sinkId, err) - return fmt.Errorf("persist initial snapshot offset: %w", err) + return 0, fmt.Errorf("persist initial snapshot offset: %w", err) } startFrom = time.Unix(0, snapshotTsNs) // walk once per process; retries resume from the persisted checkpoint @@ -218,9 +228,74 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti processEventFn = genProcessFunction(sourcePath, targetPath, excludePaths, reExcludeFileName, excludeFileNames, excludePathPatterns, dataSink, *backupOption.doDeleteFiles, debug) } - processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error { - glog.V(0).Infof("backup %s progressed to %v %0.2f/sec", sourceFiler, time.Unix(0, lastTsNs), float64(counter)/float64(3)) - return setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, lastTsNs) + if *backupOption.disableErrorRetry { + // TrivialOnError semantics: log the failure and advance. Left to the + // processor a permanent failure would pin the checkpoint and replay + // forever, which is the opposite of what this flag asks for. + inner := processEventFn + processEventFn = func(resp *filer_pb.SubscribeMetadataResponse) error { + if err := inner(resp); err != nil { + glog.Errorf("process %v: %v", resp, err) + } + return nil + } + } + + // Serial event handling caps throughput at one fetch+write at a time, so a + // busy source keeps outrunning the backup. The metadata processor fans + // events out while serializing same-path and subtree-barrier conflicts, + // and the checkpoint only advances on the contiguous processed watermark. + concurrency := *backupOption.concurrency + if concurrency < 1 || concurrency > 1024 { + glog.Warningf("invalid concurrency value %d, using default: %d", concurrency, DefaultConcurrencyLimit) + concurrency = DefaultConcurrencyLimit + } + processor := NewMetadataProcessor(processEventFn, concurrency, startFrom.UnixNano()) + + // AddOffsetFunc only checkpoints while events arrive; persist the + // completed watermark on a ticker too so work finishing on a quiet + // stream does not sit uncheckpointed until the next event. + var lastPersistedTsNs atomic.Int64 + lastPersistedTsNs.Store(-1) + stopPersist := make(chan struct{}) + persistDone := make(chan struct{}) + go func() { + defer close(persistDone) + ticker := time.NewTicker(3 * time.Second) + defer ticker.Stop() + persist := func() { + if ts := resumeTsNsFor(processor); ts != lastPersistedTsNs.Load() { + if err := setOffset(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, ts); err == nil { + lastPersistedTsNs.Store(ts) + } else { + glog.Warningf("persist backup offset: %v", err) + } + } + } + for { + select { + case <-stopPersist: + persist() + return + case <-ticker.C: + persist() + } + } + }() + + var lastLogTsNs = time.Now().UnixNano() + processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error { + processor.AddSyncJob(resp) + return nil + }, 3*time.Second, func(counter int64, lastTsNs int64) error { + offsetTsNs := processor.processedTsWatermark.Load() + if offsetTsNs == 0 { + return nil + } + now := time.Now().UnixNano() + glog.V(0).Infof("backup %s received %0.2f events/sec, processed through %v", sourceFiler, float64(counter)/(float64(now-lastLogTsNs)/1e9), time.Unix(0, offsetTsNs)) + lastLogTsNs = now + return nil }) if dataSink.IsIncremental() && *filerBackupOptions.retentionDays > 0 { @@ -256,9 +331,29 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti StartTsNs: startFrom.UnixNano(), StopTsNs: 0, EventErrorType: eventErrorType, + GetResumeTsNs: func() int64 { + return resumeTsNsFor(processor) + }, + Resubscribe: processor.ResubscribeCh(), } - return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset) + err := pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset) + // Jobs admitted before the stream ended keep running; drain them so the + // retry loop's replay cannot race an older write past a newer one. + processor.Drain() + close(stopPersist) + // setOffset has no context, so a stalled checkpoint RPC must not hold the + // retry loop: bound the wait and let a late write land in the background. + select { + case <-persistDone: + case <-time.After(30 * time.Second): + glog.Warningf("backup %s offset checkpoint did not finish in 30s; reconnecting anyway", sourceFiler) + } + // Resume the next attempt at the floored watermark — never recompute a + // position that skips events. The floor sits one tick below the earliest + // failure and stays there even when a redelivery clears the pin, so + // events dropped while the processor was stopped still replay. + return resumeTsNsFor(processor), err } diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 569ef62430..7ef16cd284 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -48,7 +48,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo now := time.Now().UnixNano() glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, offsetTsNs) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, resumeTsNsFor(processor)) }) option.clientEpoch++ @@ -65,7 +65,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), } diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index d36147191f..90d7514222 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -55,7 +55,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour now := time.Now().UnixNano() glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, time.Unix(0, offsetTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9)) lastLogTsNs = now - return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, offsetTsNs) + return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, resumeTsNsFor(processor)) }) option.clientEpoch++ @@ -77,7 +77,7 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), } diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 529472139e..554f5e3f99 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -7,6 +7,7 @@ import ( "os" "regexp" "strings" + "sync" "sync/atomic" "time" @@ -222,7 +223,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool { if state == nil || state.processor == nil { return } - offsetTsNs := state.processor.processedTsWatermark.Load() + offsetTsNs := resumeTsNsFor(state.processor) if offsetTsNs == 0 { return } @@ -475,7 +476,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi lastProgressedTsNs = offsetTsNs // collect synchronous offset statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(offsetTsNs)) - return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, offsetTsNs) + return setOffset(targetGrpcDialOption, targetFiler, signaturePrefix, sourceFilerSignature, resumeTsNsFor(processor)) }) prefix := sourcePath @@ -495,7 +496,7 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, sourceGrpcDi StopTsNs: 0, EventErrorType: pb.RetryForeverOnError, GetResumeTsNs: func() int64 { - return processor.processedTsWatermark.Load() + return resumeTsNsFor(processor) }, Resubscribe: processor.ResubscribeCh(), // While the source has only read activity it emits no metadata events, so @@ -576,16 +577,77 @@ func getOffsetWithFallback(grpcDialOption grpc.DialOption, filer pb.ServerAddres return lastOffsetTsNs, err } -func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error { - return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { +// A checkpoint write can outlive its caller's wait and land beside the next +// subscription attempt's writes; issued concurrently they may be applied out +// of order, moving saved progress backward or forward past events still owed +// a replay. Writes to the same persisted key are serialized and each is +// ordered by issuance: a write issued before the last one to land is stale +// and dropped. Deliberate regressions — resume floors, operator seeds — are +// issued after the writes they supersede, so they still apply. State is keyed +// by target filer so unrelated streams — or the two directions of a +// bidirectional sync — never block or shadow each other. +var offsetCheckpointWrites = struct { + sync.Mutex + byKey map[string]*offsetCheckpoint +}{byKey: map[string]*offsetCheckpoint{}} - syncKey := []byte(signaturePrefix + "____") - util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) +type offsetCheckpoint struct { + sync.Mutex + lastSeq int64 +} + +// resumeTsNsFor floors the processed watermark by the earliest failure the +// processor saw: a redelivery can clear the pin, but events dropped while it +// was stopped only replay if the resume position stays behind it. It also +// stays one tick below the newest timestamp seen — a watermark on the +// frontier can have finished every job it admitted while more events at the +// same timestamp still sit unread in the log, and an exclusive resume there +// would skip them. +func resumeTsNsFor(p *MetadataProcessor) int64 { + ts := p.processedTsWatermark.Load() + if newestTsNs := p.NewestSeenTsNs(); newestTsNs > 0 && ts >= newestTsNs { + ts = newestTsNs - 1 + } + if failedTsNs := p.EarliestFailedTsNs(); failedTsNs > 0 && failedTsNs-1 < ts { + ts = failedTsNs - 1 + } + return ts +} + +// offsetWriteSeq numbers checkpoint writes in issuance order so a write that +// lands late can be told apart from the fresher write it would shadow. +var offsetWriteSeq atomic.Int64 + +func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error { + syncKey := []byte(signaturePrefix + "____") + util.Uint32toBytes(syncKey[len(signaturePrefix):len(signaturePrefix)+4], uint32(signature)) + key := string(filer) + "\x00" + string(syncKey) + + offsetCheckpointWrites.Lock() + checkpoint := offsetCheckpointWrites.byKey[key] + if checkpoint == nil { + checkpoint = &offsetCheckpoint{} + offsetCheckpointWrites.byKey[key] = checkpoint + } + offsetCheckpointWrites.Unlock() + + seq := offsetWriteSeq.Add(1) + checkpoint.Lock() + defer checkpoint.Unlock() + if seq < checkpoint.lastSeq { + return nil + } + checkpoint.lastSeq = seq + + return pb.WithFilerClient(false, signature, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error { valueBuf := make([]byte, 8) util.Uint64toBytes(valueBuf, uint64(offsetTsNs)) - resp, err := client.KvPut(context.Background(), &filer_pb.KvPutRequest{ + // a stalled KvPut must not hold a caller's drain or persistence loop + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + resp, err := client.KvPut(ctx, &filer_pb.KvPutRequest{ Key: syncKey, Value: valueBuf, }) @@ -600,7 +662,6 @@ func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signature return nil }) - } func genProcessFunction(sourcePath string, targetPath string, excludePaths []string, reExcludeFileName *regexp.Regexp, excludeFileNames []*wildcard.WildcardMatcher, excludePathPatterns []*wildcard.WildcardMatcher, dataSink sink.ReplicationSink, doDeleteFiles bool, debug bool) func(resp *filer_pb.SubscribeMetadataResponse) error { diff --git a/weed/command/filer_sync_jobs.go b/weed/command/filer_sync_jobs.go index 69554fd9a4..a791158b40 100644 --- a/weed/command/filer_sync_jobs.go +++ b/weed/command/filer_sync_jobs.go @@ -137,6 +137,19 @@ type MetadataProcessor struct { failedSticky bool oldestFailedTsNs int64 + // earliestFailedTsNs is the smallest timestamp this processor ever failed + // on, latched for its lifetime: a redelivery can clear failedTs, but by + // then the processor is stopped and has already dropped unrecorded + // events — possibly sharing the failure's timestamp — so a resume + // position must still sit behind it. + earliestFailedTsNs int64 + + // newestSeenTsNs is the largest TsNs the stream has offered, job or + // marker. A watermark equal to it is not a safe resume point: more + // events at the same timestamp can still sit unread in the log, so the + // persisted position has to stay one tick below it. + newestSeenTsNs int64 + // resubscribeCh closes once a failure has stopped the processor and all // in-flight jobs have drained, asking the metadata follower to drop the // stream so the caller's reconnect replays the pinned events in order — @@ -199,6 +212,21 @@ func (t *MetadataProcessor) OldestFailedTsNs() int64 { return t.oldestFailedTsNs } +// EarliestFailedTsNs returns the smallest timestamp this processor has ever +// failed on, including failures later cleared by a redelivery. +func (t *MetadataProcessor) EarliestFailedTsNs() int64 { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + return t.earliestFailedTsNs +} + +// NewestSeenTsNs returns the largest TsNs the stream has offered so far. +func (t *MetadataProcessor) NewestSeenTsNs() int64 { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + return t.newestSeenTsNs +} + // ResubscribeCh closes once a job failure has stopped the processor and its // in-flight jobs have drained, signaling the metadata follower to drop the // stream so a reconnect replays what the watermark still covers. @@ -206,6 +234,18 @@ func (t *MetadataProcessor) ResubscribeCh() <-chan struct{} { return t.resubscribeCh } +// Drain blocks until every admitted job has finished. Callers whose stream +// just ended use it before reconnecting: a replayed event must not race a +// still-running job for the same path, or the older write could finish last +// and clobber the newer one. +func (t *MetadataProcessor) Drain() { + t.activeJobsLock.Lock() + defer t.activeJobsLock.Unlock() + for t.activeJobCount > 0 { + t.activeJobsCond.Wait() + } +} + // pathAncestors returns all proper ancestor directories of p. // For "/a/b/c", returns ["/a/b", "/a", "/"]. func pathAncestors(p util.FullPath) []util.FullPath { @@ -339,6 +379,9 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) // to it so idle stretches still advance the resume point. t.activeJobsLock.Lock() defer t.activeJobsLock.Unlock() + if resp.TsNs > t.newestSeenTsNs { + t.newestSeenTsNs = resp.TsNs + } if resp.TsNs > t.filteredTsNs { t.filteredTsNs = resp.TsNs } @@ -354,6 +397,10 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) t.activeJobsLock.Lock() defer t.activeJobsLock.Unlock() + if resp.TsNs > t.newestSeenTsNs { + t.newestSeenTsNs = resp.TsNs + } + p, newPath, kind := extractJobInfo(resp) eventKey := failedEventKey{tsNs: resp.TsNs, path: p, newPath: newPath, kind: kind} @@ -425,6 +472,9 @@ func (t *MetadataProcessor) AddSyncJob(resp *filer_pb.SubscribeMetadataResponse) // whole mechanism exists to remove. t.stopped = true t.activeJobsCond.Broadcast() + if t.earliestFailedTsNs == 0 || resp.TsNs < t.earliestFailedTsNs { + t.earliestFailedTsNs = resp.TsNs + } if t.failedSticky { if resp.TsNs < t.oldestFailedTsNs { t.oldestFailedTsNs = resp.TsNs diff --git a/weed/command/filer_sync_jobs_test.go b/weed/command/filer_sync_jobs_test.go index 968858638a..20f5ab04da 100644 --- a/weed/command/filer_sync_jobs_test.go +++ b/weed/command/filer_sync_jobs_test.go @@ -4,6 +4,7 @@ import ( "container/heap" "errors" "fmt" + "sync/atomic" "testing" "time" @@ -553,6 +554,61 @@ func TestSamePathBarrierSerialization(t *testing.T) { }) } +// TestConcurrentJobsUpToLimit proves independent-path jobs run in parallel up +// to the concurrency limit: every admitted job is in flight before any +// finishes (a serial loop could only have one running), and admissions past +// the limit wait for a slot rather than running ahead of it. +func TestConcurrentJobsUpToLimit(t *testing.T) { + const limit = 3 + started := make(chan struct{}, limit) + release := make(chan struct{}) + var inFlight, maxInFlight atomic.Int32 + + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + cur := inFlight.Add(1) + for { + if old := maxInFlight.Load(); cur <= old || maxInFlight.CompareAndSwap(old, cur) { + break + } + } + started <- struct{}{} + <-release + inFlight.Add(-1) + return nil + }, limit, 0) + + admitDone := make(chan struct{}) + go func() { + defer close(admitDone) + for i := 0; i < limit+1; i++ { + p.AddSyncJob(makeResp("/d", fmt.Sprintf("f%d", i), false, int64(i+1), true)) + } + }() + + for i := 0; i < limit; i++ { + select { + case <-started: + case <-time.After(10 * time.Second): + t.Fatalf("only %d of %d jobs entered flight", i, limit) + } + } + // The extra admission must still be blocked on a full pool. + select { + case <-started: + t.Fatal("a job started with no free slot") + case <-admitDone: + t.Fatal("submission returned past the concurrency limit") + case <-time.After(200 * time.Millisecond): + } + + close(release) + <-admitDone + p.Drain() + if got := maxInFlight.Load(); got != limit { + t.Fatalf("peak in-flight = %d, want %d", got, limit) + } +} + // benchResult prevents the compiler from optimizing away the conflict check. var benchResult bool @@ -1073,3 +1129,107 @@ func TestResubscribeWaitsForSameTsSibling(t *testing.T) { t.Fatal("resubscribe channel never closed after the same-ts jobs drained") } } + +// TestResumeStaysBehindClearedPin verifies the resume floor outlives the +// ledger pin: a redelivery can clear failedTs, but events dropped while the +// processor was stopped — including ones sharing the failure's timestamp — +// still have to replay, so the resume position stays one tick below the +// earliest failure. +func TestResumeStaysBehindClearedPin(t *testing.T) { + fail := true + release := make(chan struct{}) + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + name := resp.EventNotification.NewEntry.GetName() + if name == "slow.txt" { + <-release + } + if name == "bad.txt" && fail { + return errors.New("AccessDenied: Access Denied") + } + return nil + }, 10, 0) + + // the slow job holds the drain — and the resubscribe — open so the + // redelivery still lands on this processor + p.AddSyncJob(makeResp("/dir", "slow.txt", false, 300, true)) + p.AddSyncJob(makeResp("/dir", "bad.txt", false, 200, true)) + deadline := time.Now().Add(10 * time.Second) + for p.OldestFailedTsNs() != 200 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + + // a same-ts sibling arriving after the stop is dropped + p.AddSyncJob(makeResp("/dir", "sib.txt", false, 200, true)) + p.activeJobsLock.Lock() + dropped := p.activeJobTs[200] == 0 + p.activeJobsLock.Unlock() + if !dropped { + t.Fatal("sibling admitted after the failure stopped the processor") + } + + // the redelivery runs and clears the pin + fail = false + p.AddSyncJob(makeResp("/dir", "bad.txt", false, 200, true)) + deadline = time.Now().Add(10 * time.Second) + for p.OldestFailedTsNs() != 0 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := p.OldestFailedTsNs(); got != 0 { + t.Fatalf("oldest failed = %d after the redelivery, want 0", got) + } + + if got := p.EarliestFailedTsNs(); got != 200 { + t.Fatalf("earliest failed = %d, want the floor held at 200", got) + } + if got := resumeTsNsFor(p); got != 199 { + t.Fatalf("resume = %d, want 199 so the dropped sibling replays", got) + } + + close(release) + waitForJobsToDrain(t, p) +} + +// TestResumeStaysBelowNewestSeen covers the frontier case: when every admitted +// job has finished, the watermark sits on the newest timestamp seen — but the +// log can still hold same-timestamp events the stream has not delivered yet. +// An exclusive resume there would skip them, so the position stays one tick +// below until a newer timestamp proves the block is closed. +func TestResumeStaysBelowNewestSeen(t *testing.T) { + p := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + return nil + }, 10, 0) + + // the event at 100 finishes before a same-timestamp sibling could even be + // read; the watermark reaches the frontier but cannot persist there + p.AddSyncJob(makeResp("/dir", "a.txt", false, 100, true)) + waitForJobsToDrain(t, p) + if got := resumeTsNsFor(p); got != 99 { + t.Fatalf("resume = %d at the frontier, want 99 so an unread same-ts event replays", got) + } + + // once a newer timestamp is seen the 100-block is closed: a slow job at + // 200 in flight pins nothing below it, so 100 is now a safe position + release := make(chan struct{}) + p2 := NewMetadataProcessor(func(resp *filer_pb.SubscribeMetadataResponse) error { + if resp.TsNs == 200 { + <-release + } + return nil + }, 10, 0) + p2.AddSyncJob(makeResp("/dir", "c.txt", false, 100, true)) + p2.AddSyncJob(makeResp("/dir", "b.txt", false, 200, true)) + deadline := time.Now().Add(10 * time.Second) + for p2.processedTsWatermark.Load() != 100 && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := resumeTsNsFor(p2); got != 100 { + t.Fatalf("resume = %d below the frontier, want 100", got) + } + + // finishing the 200 job lands the watermark back on the frontier + close(release) + waitForJobsToDrain(t, p2) + if got := resumeTsNsFor(p2); got != 199 { + t.Fatalf("resume = %d at the frontier, want 199", got) + } +} diff --git a/weed/command/filer_sync_offset_test.go b/weed/command/filer_sync_offset_test.go index 835bb70fa0..93caf23e5c 100644 --- a/weed/command/filer_sync_offset_test.go +++ b/weed/command/filer_sync_offset_test.go @@ -1,6 +1,10 @@ package command -import "testing" +import ( + "testing" + + "github.com/seaweedfs/seaweedfs/weed/util" +) // The offset key is a persistence contract: a changed form orphans saved // checkpoints, so the historical forms must survive verbatim and the @@ -74,3 +78,50 @@ func TestSyncOffset_HistoricalFallbackThenIndependence(t *testing.T) { t.Fatalf("B read = (%d, %v), want (111, nil)", got, err) } } + +// TestSyncOffset_StaleWriteDropped: a checkpoint write issued before a write +// that already landed under the same key is stale and must not apply, while +// a deliberately regressed value issued later — a resume floor or an +// operator seed — still does. +func TestSyncOffset_StaleWriteDropped(t *testing.T) { + filerAddr, dial := startKvFiler(t) + prefix := getSignaturePrefixByPath("/data", "/backup-stale") + sourceSig := int32(77) + + if err := setOffset(dial, filerAddr, prefix, sourceSig, 300); err != nil { + t.Fatalf("seed offset: %v", err) + } + + // Pin the checkpoint past the next write's issuance seq, simulating a + // write that was issued before a newer one already landed. + syncKey := []byte(prefix + "____") + util.Uint32toBytes(syncKey[len(prefix):len(prefix)+4], uint32(sourceSig)) + mapKey := string(filerAddr) + "\x00" + string(syncKey) + offsetCheckpointWrites.Lock() + checkpoint := offsetCheckpointWrites.byKey[mapKey] + offsetCheckpointWrites.Unlock() + if checkpoint == nil { + t.Fatal("no checkpoint state for the seeded key") + } + checkpoint.Lock() + checkpoint.lastSeq = offsetWriteSeq.Load() + 1000 + checkpoint.Unlock() + + if err := setOffset(dial, filerAddr, prefix, sourceSig, 100); err != nil { + t.Fatalf("stale write: %v", err) + } + if got, err := getOffset(dial, filerAddr, prefix, sourceSig); err != nil || got != 300 { + t.Fatalf("after stale write = (%d, %v), want (300, nil)", got, err) + } + + // a later-issued regression is deliberate and must apply + checkpoint.Lock() + checkpoint.lastSeq = 0 + checkpoint.Unlock() + if err := setOffset(dial, filerAddr, prefix, sourceSig, 199); err != nil { + t.Fatalf("floored write: %v", err) + } + if got, err := getOffset(dial, filerAddr, prefix, sourceSig); err != nil || got != 199 { + t.Fatalf("after floored write = (%d, %v), want (199, nil)", got, err) + } +} From 673d1a02e059919a40b6e4fbcf7d88f04d4114fa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 20:34:31 -0700 Subject: [PATCH 36/70] operation: euthanize pooled upload conns that stall far below a fast host (#12003) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * operation: euthanize pooled upload conns that stall far below a fast host A pooled keep-alive conn can wedge and drain the request body at a trickle while fresh conns to the same volume run at full rate; every later upload routed onto it stalls for tens of seconds, indefinitely. Track per-host healthy-rate uploads (>=1MiB/s on a >=256KiB body) and, on a host that was fast within the last minute, cancel an upload still in flight past max(10s, size/256KiB/s): the transport discards the wedged conn and the existing retry dials a fresh one. A watchdog kill also clears the fast mark so the retried attempt runs unbounded, and hosts that never prove fast keep today's unbounded behavior - a chronically slow link is never penalized. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * operation: keep the stall watchdog armed through the response read uploadDone was set as soon as Do() returned, so a conn that accepted the request body then wedged on the reply escaped the watchdog and could park the upload for hours — the same failure this watchdog exists to kill. Deferring the done-store keeps the timer live until the function returns, so a stalled io.ReadAll is cancelled and retried like a stalled write. The redundant post_err recheck goes with it. * operation: pin watchdog coverage of a stalled response read * operation: disarm the stall watchdog for an upload's remaining attempts Once the watchdog has cancelled an attempt, a concurrent healthy upload to the same host can restore the fast mark during the retry backoff. The re-armed watchdog would then cancel the retry too, and a large upload that merely exceeds the floor could burn every attempt. Track the watchdog kill on uploadStatusError and carry a stallWatchdogDisarmed flag through retriedUploadData so the rest of that upload's attempts run unbounded; other uploads to the host keep their watchdog. * operation: pin watchdog disarm on retry, fresh-dial, and slow-upload cases * operation: drop the send-stall watchdog once the request body drains A wedged pooled conn is only blameworthy while it is still trickling the request body; once the body is on the wire the wait is server-side commit, which the transport's response header timeout already bounds. Cancelling then would fail a healthy write whose replication or fsync is simply slow. Track body EOF through a wrapping reader, gate the send-side timer on it, and re-arm the watchdog only around the response-body read, which the header timeout does not cover. * operation: prove a host fast only on a verified upload A quick rejection moves bytes no faster than the wire did, so mark the host from the body-send duration and only after the response checks out. * operation: count an abandoned request body as sent The transport closes a request body it never finished draining when the server answers early, so EOF alone under-reports the send boundary and the stall watchdog could still blame the conn during the reply wait. * operation: scope the body-sent mark to the current attempt A lagging Close or EOF from the first attempt's body could stamp the shared mark after the inner retry reset it, disarming the watchdog for a send that never happened; each attempt now points the watchdog at its own sent timestamp. * operation: drop only the fast mark the stall timer observed A concurrent upload can re-prove the host while a stalled attempt is in flight; deleting the key outright would erase that newer mark, so the watchdog now removes only the value it armed against. * operation: keep the stall disarm on a per-call upload option retriedUploadData marked the caller's UploadOption, so a caller reusing it across UploadData calls would start the next upload with the watchdog off and lose the cancellation this adds. * test: stall watchdog retry must fail if the conn gets cancelled The fake retry slept through any cancellation, so a re-armed watchdog firing mid-retry still let the upload succeed — the guard was untested. Park on the request context like a real wedged conn so a re-armed watchdog cancels the attempt and fails the test. * operation: prove the fast-host mark with elapsed-to-response Body-drain time only measures when the transport consumed the reader: kernel socket buffers can swallow a small body instantly on a slow link, inflating the rate and marking the host fast. The next upload then gets its conn cancelled mid-send despite making progress. Elapsed-to-response bounds remote speed from below — the server cannot answer before the body lands — so the mark can only under-mark, never over-mark. * operation: scale the stall budget off the host's measured rate Fixed bps limits fit no link well: 256KiB/s never fires on a slow link and waits far too long on a fast one. Record the wire rate of each sizeable upload and let a later upload run for slowFactor times the body's span at that rate — a wedged conn shows as an order-of-magnitude drop on any link speed. The recorded rate uses the wire body size so a pre-compressed payload no longer inflates the mark. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/operation/upload_content.go | 202 ++++++++++++-- weed/operation/upload_content_test.go | 368 ++++++++++++++++++++++++++ 2 files changed, 549 insertions(+), 21 deletions(-) diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go index 704f70f49d..081d295e66 100644 --- a/weed/operation/upload_content.go +++ b/weed/operation/upload_content.go @@ -17,6 +17,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/seaweedfs/seaweedfs/weed/util/request_id" @@ -63,6 +64,10 @@ type UploadOption struct { MaxAttempts int // <=0 uses the default GenUploadUrl func(host, fileId string) string // if nil → fallback "http://{host}/{fileId}" FilerJwt security.FilerJwtProvider // credential for proxy chunk URLs; nil → process-wide jwt.filer_signing + + // stallWatchdogDisarmed turns the watchdog off for this upload's later + // attempts once it fired: a re-arm could burn every retry. + stallWatchdogDisarmed bool } type UploadResult struct { @@ -125,6 +130,7 @@ var ( // StatusCode 0 means the request never got a response — a transport failure. type uploadStatusError struct { StatusCode int + stalled bool // the stall watchdog cancelled this attempt's request err error } @@ -163,6 +169,11 @@ type HTTPClient interface { // Uploader type Uploader struct { httpClient HTTPClient + // fastUploadHosts records the wire rate of the last sizeable upload to + // each host, so a later upload stalling far below it can be blamed on + // that one connection rather than the link and get cancelled for a retry + // on a fresh conn. + fastUploadHosts sync.Map // host:port → fastHostMark } func NewUploader() (*Uploader, error) { @@ -328,6 +339,11 @@ func (uploader *Uploader) doUpload(ctx context.Context, reader io.Reader, option } func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, option *UploadOption) (uploadResult *UploadResult, err error) { + // A stall disarms the watchdog for this upload's remaining attempts only; + // keep that on a per-call option so a caller reusing UploadOption starts + // the next upload armed. + localOption := *option + option = &localOption maxAttempts := option.MaxAttempts if maxAttempts <= 0 { maxAttempts = 3 @@ -348,6 +364,10 @@ func (uploader *Uploader) retriedUploadData(ctx context.Context, data []byte, op if ctx.Err() != nil { return nil, ctx.Err() } + var statusErr *uploadStatusError + if errors.As(err, &statusErr) && statusErr.stalled { + option.stallWatchdogDisarmed = true + } glog.WarningfCtx(ctx, "uploading %d to %s: %v", i, option.UploadUrl, err) } return @@ -415,14 +435,15 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option _, err = w.Write(encryptedData) return }, len(encryptedData), &UploadOption{ - UploadUrl: option.UploadUrl, - Filename: "", - Cipher: false, - IsInputCompressed: false, - MimeType: "", - PairMap: nil, - Jwt: option.Jwt, - BytesBuffer: option.BytesBuffer, + UploadUrl: option.UploadUrl, + Filename: "", + Cipher: false, + IsInputCompressed: false, + MimeType: "", + PairMap: nil, + Jwt: option.Jwt, + BytesBuffer: option.BytesBuffer, + stallWatchdogDisarmed: option.stallWatchdogDisarmed, }) if uploadResult == nil { return @@ -443,15 +464,16 @@ func (uploader *Uploader) doUploadData(ctx context.Context, data []byte, option _, err = w.Write(data) return }, len(data), &UploadOption{ - UploadUrl: option.UploadUrl, - Filename: option.Filename, - Cipher: false, - IsInputCompressed: contentIsGzipped, - MimeType: option.MimeType, - PairMap: option.PairMap, - Jwt: option.Jwt, - Md5: option.Md5, - BytesBuffer: option.BytesBuffer, + UploadUrl: option.UploadUrl, + Filename: option.Filename, + Cipher: false, + IsInputCompressed: contentIsGzipped, + MimeType: option.MimeType, + PairMap: option.PairMap, + Jwt: option.Jwt, + Md5: option.Md5, + BytesBuffer: option.BytesBuffer, + stallWatchdogDisarmed: option.stallWatchdogDisarmed, }) if uploadResult == nil { return @@ -514,7 +536,14 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction } else { reqReader = bytes.NewReader(option.BytesBuffer.Bytes()) } - req, postErr := http.NewRequestWithContext(ctx, http.MethodPost, option.UploadUrl, reqReader) + // bodySize is what the conn must move: originalDataSize counts bytes + // before compression, which would inflate both the recorded rate and the + // stall budget for a pre-compressed body. + bodySize := int64(reqReader.Len()) + reqCtx, cancelUpload := context.WithCancel(ctx) + defer cancelUpload() + + req, postErr := http.NewRequestWithContext(reqCtx, http.MethodPost, option.UploadUrl, reqReader) if postErr != nil { glog.V(1).InfofCtx(ctx, "create upload request %s: %v", option.UploadUrl, postErr) return nil, fmt.Errorf("create upload request %s: %v", option.UploadUrl, postErr) @@ -529,7 +558,46 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction request_id.InjectToRequest(ctx, req) + var uploadDone atomic.Bool + defer uploadDone.Store(true) + // bodySentAt marks when the transport finished draining the request body; + // past it the wait is server-side commit, which the transport's response + // header timeout already bounds — the stall watchdog only blames a conn + // that is still trickling the body. + var curBodySentAt atomic.Pointer[atomic.Int64] + bodySentAt := &atomic.Int64{} + curBodySentAt.Store(bodySentAt) + req.Body = &watchdogBody{ReadCloser: req.Body, sentAt: bodySentAt} + mark, hostWasFast := uploader.fastUploadHosts.Load(req.URL.Host) + hostWasFast = hostWasFast && !option.stallWatchdogDisarmed && time.Since(time.Unix(0, mark.(fastHostMark).at)) < fastUploadHostMemory + stallFired := func() { + if !uploadDone.CompareAndSwap(false, true) { + return + } + // The host was moving bodies at full rate a moment ago, so an + // upload still trickling far below that rate rides a wedged pooled + // conn; cancel drops the conn so the retry dials a fresh one. + // CompareAndDelete leaves a newer mark a concurrent success stored. + uploader.fastUploadHosts.CompareAndDelete(req.URL.Host, mark) + glog.WarningfCtx(ctx, "upload %d bytes to %s stalled on a pooled connection, retrying on a fresh one", originalDataSize, option.UploadUrl) + cancelUpload() + } + var stallTimer *time.Timer + if hostWasFast { + defer func() { + if stallTimer != nil { + stallTimer.Stop() + } + }() + stallTimer = time.AfterFunc(uploadStallTimeout(bodySize, mark.(fastHostMark).bps), func() { + if curBodySentAt.Load().Load() == 0 { + stallFired() + } + }) + } + // print("+") + started := time.Now() resp, post_err := uploader.httpClient.Do(req) defer util_http.CloseResponse(resp) if post_err != nil { @@ -548,7 +616,13 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction // "connection reset" error with a misleading "Body length 0". if req.GetBody != nil { if newBody, gbErr := req.GetBody(); gbErr == nil { - req.Body = newBody + bodySentAt = &atomic.Int64{} + curBodySentAt.Store(bodySentAt) + started = time.Now() + if stallTimer != nil && !uploadDone.Load() { + stallTimer.Reset(uploadStallTimeout(bodySize, mark.(fastHostMark).bps)) + } + req.Body = &watchdogBody{ReadCloser: newBody, sentAt: bodySentAt} resp, post_err = uploader.httpClient.Do(req) defer util_http.CloseResponse(resp) } else { @@ -561,22 +635,40 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction } if post_err != nil { stats.UploadErrorCounter.WithLabelValues("0").Inc() - return nil, &uploadStatusError{StatusCode: 0, err: fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)} + return nil, &uploadStatusError{StatusCode: 0, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("upload %s %d bytes to %v: %v", option.Filename, originalDataSize, option.UploadUrl, post_err)} } + // Rate is bounded by elapsed-to-response: the server cannot answer before + // the body lands, so this can only understate link speed — a drain-time + // reading mistakes kernel-buffer absorption for wire speed and would mark + // a slow link fast. The mark still only lands on a verified upload: a + // fast rejection says nothing about how the conn drains. + movedIn := time.Since(started) + // print("-") var ret UploadResult etag := getEtag(resp) if resp.StatusCode == http.StatusNoContent { + if bps := uploadObservedBps(bodySize, movedIn); bps > 0 { + uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps}) + } ret.ETag = etag ret.ContentMd5 = resp.Header.Get("Content-MD5") return &ret, nil } + if stallTimer != nil { + // The header timeout does not cover the response body: keep the + // watchdog armed for the read so a conn that wedges mid-reply gets + // cancelled for a fresh retry. + respStallTimer := time.AfterFunc(uploadStallMinWait, stallFired) + defer respStallTimer.Stop() + } resp_body, ra_err := io.ReadAll(resp.Body) + uploadDone.Store(true) if ra_err != nil { stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() - return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} + return nil, &uploadStatusError{StatusCode: resp.StatusCode, stalled: reqCtx.Err() != nil && ctx.Err() == nil, err: fmt.Errorf("read response body %v: %w", option.UploadUrl, ra_err)} } unmarshal_err := json.Unmarshal(resp_body, &ret) @@ -589,6 +681,9 @@ func (uploader *Uploader) upload_content(ctx context.Context, fillBufferFunction stats.UploadErrorCounter.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() return nil, &uploadStatusError{StatusCode: resp.StatusCode, err: fmt.Errorf("unmarshalled error %v: %v", option.UploadUrl, ret.Error)} } + if bps := uploadObservedBps(bodySize, movedIn); bps > 0 { + uploader.fastUploadHosts.Store(req.URL.Host, fastHostMark{at: time.Now().UnixNano(), bps: bps}) + } ret.ETag = etag ret.ContentMd5 = resp.Header.Get("Content-MD5") return &ret, nil @@ -601,3 +696,68 @@ func getEtag(r *http.Response) (etag string) { } return } + +// uploadRateMinBytes is the smallest body whose measured rate can mark a +// host: smaller uploads are dominated by round-trip and commit overhead, not +// wire speed. +const uploadRateMinBytes = int64(256 << 10) + +var ( + // uploadStallSlowFactor is how far below the host's last observed rate an + // upload may run before its conn is presumed wedged: at 8, a host that + // just moved a body at 8MiB/s tolerates down to 1MiB/s. + uploadStallSlowFactor = int64(8) + uploadStallMinWait = 10 * time.Second + fastUploadHostMemory = time.Minute +) + +// fastHostMark remembers the wire rate of a host's last sizeable upload, so a +// later upload running far below it blames the pooled conn rather than the +// link — no fixed rate fits every deployment's links. +type fastHostMark struct { + at int64 // UnixNano + bps int64 // wire bytes/sec measured elapsed-to-response +} + +// uploadStallTimeout is the most an upload may take on a conn whose host last +// proved bps: the time the body needs at an uploadStallSlowFactor of that +// rate, with a floor so a fast conn is not raced on small bodies. +func uploadStallTimeout(size, bps int64) time.Duration { + if bps > 0 { + if d := time.Duration(size/bps) * time.Duration(uploadStallSlowFactor) * time.Second; d > uploadStallMinWait { + return d + } + } + return uploadStallMinWait +} + +// uploadObservedBps returns the wire rate a completed upload proved on its +// host, or 0 when the body was too small to measure. Elapsed-to-response can +// only understate the link's speed, which keeps the mark — and the stall +// budget derived from it — conservative. +func uploadObservedBps(size int64, elapsed time.Duration) int64 { + if size < uploadRateMinBytes || elapsed <= 0 { + return 0 + } + return int64(float64(size) / elapsed.Seconds()) +} + +// watchdogBody records when the transport drains the request body: before that +// a stuck send blames the conn; after it the wait is the server's. +type watchdogBody struct { + io.ReadCloser + sentAt *atomic.Int64 +} + +func (b *watchdogBody) Read(p []byte) (int, error) { + n, err := b.ReadCloser.Read(p) + if err == io.EOF { + b.sentAt.Store(time.Now().UnixNano()) + } + return n, err +} + +func (b *watchdogBody) Close() error { + b.sentAt.Store(time.Now().UnixNano()) + return b.ReadCloser.Close() +} diff --git a/weed/operation/upload_content_test.go b/weed/operation/upload_content_test.go index ebbfb722f4..d80c8cd591 100644 --- a/weed/operation/upload_content_test.go +++ b/weed/operation/upload_content_test.go @@ -8,10 +8,12 @@ import ( "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "strings" "sync" + "sync/atomic" "testing" "time" @@ -199,6 +201,372 @@ func (c *bodyCapturingHTTPClient) Do(req *http.Request) (*http.Response, error) }, nil } +// stallingHTTPClient parks the first stalls calls without draining the request +// body (a conn that never drains) until the request ctx is cancelled, and +// delays the rest by slowFor after draining. +type stallingHTTPClient struct { + mu sync.Mutex + stalls int + slowFor time.Duration + requestCtxs []context.Context +} + +func (c *stallingHTTPClient) Do(req *http.Request) (*http.Response, error) { + c.mu.Lock() + c.requestCtxs = append(c.requestCtxs, req.Context()) + stall := c.stalls > 0 + if stall { + c.stalls-- + } + c.mu.Unlock() + + if stall { + <-req.Context().Done() + return nil, req.Context().Err() + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + if c.slowFor > 0 { + time.Sleep(c.slowFor) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"size":1}`)), + }, nil +} + +// markHostFast arms the stall watchdog the way a recent fast upload would: +// the recorded rate is high enough that the adaptive budget lands on +// uploadStallMinWait for test-sized bodies. +func markHostFast(u *Uploader, host string) { + u.fastUploadHosts.Store(host, fastHostMark{at: time.Now().UnixNano(), bps: 1 << 30}) +} + +// A host that just served a fast upload arms the stall watchdog: a request +// parked well past the floor is riding a wedged conn, so it is cancelled and +// the retry runs on a fresh conn without the watchdog. +func TestUploadStallWatchdogRetriesOnFreshConn(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + client := &stallingHTTPClient{stalls: 1} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried on a fresh conn: %v", err) + } + if result.RetryCount != 1 || len(client.requestCtxs) != 2 { + t.Fatalf("got retries=%d calls=%d, want one retry on a fresh conn", result.RetryCount, len(client.requestCtxs)) + } + if err := client.requestCtxs[0].Err(); err == nil { + t.Fatalf("first attempt context was not cancelled") + } +} + +// readStallHTTPClient answers with a response body that never delivers until +// the request ctx is cancelled — the conn that took the request then wedged on +// the read side. +type readStallHTTPClient struct { + mu sync.Mutex + stall bool +} + +func (c *readStallHTTPClient) Do(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + c.mu.Lock() + stall := c.stall + c.stall = false + c.mu.Unlock() + var body io.ReadCloser = io.NopCloser(strings.NewReader(`{"size":1}`)) + if stall { + body = &ctxBlockedBody{ctx: req.Context()} + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: body, + }, nil +} + +type ctxBlockedBody struct{ ctx context.Context } + +func (b *ctxBlockedBody) Read([]byte) (int, error) { + <-b.ctx.Done() + return 0, b.ctx.Err() +} + +func (b *ctxBlockedBody) Close() error { return nil } + +// The watchdog stays armed through the response read: a conn that accepts the +// body then wedges on the reply is as wedged as one that never drains. +func TestUploadStallWatchdogCoversResponseRead(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + client := &readStallHTTPClient{stall: true} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried past the stalled read: %v", err) + } + if result.RetryCount != 1 { + t.Fatalf("got retries=%d, want one retry on a fresh conn", result.RetryCount) + } +} + +// rearmHTTPClient parks the first attempt on a conn that never drains and +// re-marks the host fast as it fails — a concurrent healthy upload can restore +// the mark while this upload is between attempts. The retry drains slowly, so +// a re-armed watchdog would cancel it too. +type rearmHTTPClient struct { + uploader *Uploader + calls int32 + slowFor time.Duration +} + +func (c *rearmHTTPClient) Do(req *http.Request) (*http.Response, error) { + if atomic.AddInt32(&c.calls, 1) == 1 { + <-req.Context().Done() + markHostFast(c.uploader, req.URL.Host) + return nil, req.Context().Err() + } + // Park past the watchdog floor like a real wedged conn: a re-armed + // watchdog must cancel this attempt instead of the sleep masking it. + select { + case <-req.Context().Done(): + return nil, req.Context().Err() + case <-time.After(c.slowFor): + } + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + return &http.Response{ + StatusCode: http.StatusCreated, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"size":1}`)), + }, nil +} + +// A re-proven host must not re-arm the watchdog for the retry: the conn it +// cancelled says nothing about the next conn, and re-arming could burn every +// remaining attempt. +func TestUploadStallWatchdogStaysDisarmedOnRetry(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 40 * time.Millisecond + + uploader := newUploader(nil) + uploader.httpClient = &rearmHTTPClient{uploader: uploader, slowFor: 60 * time.Millisecond} + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("retry should run unbounded once the watchdog has fired: %v", err) + } + if result.RetryCount != 1 { + t.Fatalf("got retries=%d, want one retry", result.RetryCount) + } +} + +// The disarm is per upload: a caller reusing its UploadOption still gets an +// armed watchdog on the next call — a second stalled conn must cancel too. +func TestUploadStallWatchdogRearmsForNextUpload(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 40 * time.Millisecond + + client := &stallingHTTPClient{} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // IsInputCompressed keeps the body large enough to re-prove the host: + // compressed near-zeros would send too few bytes to re-mark it. + option := &UploadOption{UploadUrl: "http://test-host:8080/1,deadbeef", MaxAttempts: 2, IsInputCompressed: true} + for i := 0; i < 2; i++ { + client.mu.Lock() + client.stalls = 1 + client.mu.Unlock() + result, err := uploader.UploadData(ctx, make([]byte, 300*1024), option) + if err != nil { + t.Fatalf("upload %d should have retried on a fresh conn: %v", i, err) + } + if result.RetryCount != 1 { + t.Fatalf("upload %d got retries=%d, want its own stall caught", i, result.RetryCount) + } + } + if option.stallWatchdogDisarmed { + t.Fatalf("a stall must not disarm the watchdog on the caller's option") + } +} + +// A real transport must drop the conn a cancelled request was riding: the +// retry has to dial again rather than land back on the parked conn. The first +// conn here wedges mid-reply — headers flush, then the body never arrives. +func TestUploadStallWatchdogDialsFreshConn(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 50 * time.Millisecond + + var requests, dials int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.Copy(io.Discard, r.Body) + if atomic.AddInt32(&requests, 1) == 1 { + w.WriteHeader(http.StatusCreated) + w.(http.Flusher).Flush() + <-r.Context().Done() + return + } + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"size":1}`)) + })) + defer srv.Close() + + tr := &http.Transport{ + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + atomic.AddInt32(&dials, 1) + return new(net.Dialer).DialContext(ctx, network, addr) + }, + } + defer tr.CloseIdleConnections() + uploader := newUploader(&http.Client{Transport: tr}) + + host := strings.TrimPrefix(srv.URL, "http://") + markHostFast(uploader, host) + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: srv.URL + "/1,deadbeef", + MaxAttempts: 2, + }) + if err != nil { + t.Fatalf("upload should have retried on a fresh conn: %v", err) + } + if result.RetryCount != 1 || atomic.LoadInt32(&dials) != 2 { + t.Fatalf("got retries=%d dials=%d, want one retry on a fresh conn", result.RetryCount, dials) + } +} + +// Once the body drains the wait is the server's commit: a write that takes +// longer than the stall floor to acknowledge must not be cancelled — the conn +// is healthy, the server is just busy. +func TestUploadStallWatchdogAllowsSlowCommit(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 30 * time.Millisecond + + client := &stallingHTTPClient{slowFor: 80 * time.Millisecond} + uploader := newUploader(client) + markHostFast(uploader, "test-host:8080") + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err != nil || result.RetryCount != 0 { + t.Fatalf("a slow commit must not be cancelled: result=%+v err=%v", result, err) + } +} + +// A fast rejection proves nothing about how the conn drains, so it must not +// mark the host fast and arm the watchdog for the next upload. +func TestUploadStallWatchdogIgnoresFailedWrites(t *testing.T) { + uploader := newUploader(&rejectHTTPClient{}) + _, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://test-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err == nil { + t.Fatalf("rejected upload should fail") + } + if _, ok := uploader.fastUploadHosts.Load("test-host:8080"); ok { + t.Fatalf("a failed upload must not mark the host fast") + } +} + +type rejectHTTPClient struct{} + +func (c *rejectHTTPClient) Do(req *http.Request) (*http.Response, error) { + if req.Body != nil { + _, _ = io.Copy(io.Discard, req.Body) + } + return &http.Response{ + StatusCode: http.StatusInternalServerError, + Header: make(http.Header), + Body: io.NopCloser(strings.NewReader(`{"error":"volume is read only"}`)), + }, nil +} + +// Without a recent fast upload the watchdog stays off: a chronically slow link +// is the link, not a conn, and must not get its uploads cancelled. +func TestUploadStallWatchdogKeepsUnprovenHost(t *testing.T) { + defer func(orig time.Duration) { uploadStallMinWait = orig }(uploadStallMinWait) + uploadStallMinWait = 20 * time.Millisecond + + client := &stallingHTTPClient{slowFor: 60 * time.Millisecond} + uploader := newUploader(client) + + result, err := uploader.UploadData(context.Background(), make([]byte, 300*1024), &UploadOption{ + UploadUrl: "http://slow-host:8080/1,deadbeef", + MaxAttempts: 1, + }) + if err != nil { + t.Fatalf("unproven host upload should not be cancelled: %v", err) + } + if result.RetryCount != 0 || len(client.requestCtxs) != 1 { + t.Fatalf("got retries=%d calls=%d, want a single unbounded attempt", result.RetryCount, len(client.requestCtxs)) + } +} + +// Any sizeable completed upload marks its host at the rate it proved — slow +// or fast — because the stall budget scales off that rate. Only a too-small +// body fails to measure. +func TestUploadObservedBps(t *testing.T) { + if bps := uploadObservedBps(4<<20, 4*time.Second); bps != 1<<20 { + t.Fatalf("4MiB in 4s observed %d B/s, want 1MiB/s", bps) + } + if bps := uploadObservedBps(4<<20, 3*time.Hour); bps <= 0 { + t.Fatalf("a slow upload still proves the rate it moved at") + } + if bps := uploadObservedBps(64*1024, time.Millisecond); bps != 0 { + t.Fatalf("a small upload says nothing about host speed") + } + if bps := uploadObservedBps(4<<20, 0); bps != 0 { + t.Fatalf("a zero-duration sample is not a measurement") + } +} + +// The stall budget scales with the rate the host last proved, so the same +// body gets little slack on a fast link and plenty on a slow one — no fixed +// floor fits both. +func TestUploadStallTimeoutScalesWithMeasuredRate(t *testing.T) { + // 8MiB/s observed → a 64MiB body gets 8x the time it needs at that rate. + if d := uploadStallTimeout(64<<20, 8<<20); d != 64*time.Second { + t.Fatalf("stall budget = %v, want 64s", d) + } + // The same body on a 64MiB/s host gets 8s — under the floor, so clamped. + if d := uploadStallTimeout(64<<20, 64<<20); d != uploadStallMinWait { + t.Fatalf("stall budget = %v, want the min wait %v", d, uploadStallMinWait) + } + // A tiny body on a fast host cannot race its own conn. + if d := uploadStallTimeout(1, 1<<30); d != uploadStallMinWait { + t.Fatalf("stall budget = %v, want the min wait %v", d, uploadStallMinWait) + } +} + type fsyncAssignClient struct { filer_pb.SeaweedFilerClient response *filer_pb.AssignVolumeResponse From 8541f959a93da49380e19237fd09ad2dbfd122fd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 20:45:46 -0700 Subject: [PATCH 37/70] filer.conf: merge fsync on presence like worm (#11953) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * filer.conf: merge fsync on presence like worm fsync is a pure preference, so OR-merging it meant a nested rule could never turn a parent's fsync off. Make PathConf.fsync optional and merge it most-specific-wins, matching worm: an unset rule inherits, an explicit value overrides in either direction. readOnly and disableChunkDeletion stay OR'ed — they are fail-closed locks quota enforcement depends on. Pre-version-2 configs were written with EmitUnpopulated, so every rule carries an explicit fsync=false that meant nothing; it is dropped to unset on load so it does not read as an opt-out. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.conf: test fsync override survives a save/reload round trip --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- other/java/client/src/main/proto/filer.proto | 3 +- weed/filer/filer.go | 2 +- weed/filer/filer_conf.go | 33 ++++++---- weed/filer/filer_conf_test.go | 69 +++++++++++++++++++- weed/pb/filer.proto | 3 +- weed/pb/filer_pb/filer.pb.go | 46 ++++++------- weed/pb/filer_pb/filer_grpc.pb.go | 2 +- weed/pb/filer_pb/filer_vtproto.pb.go | 42 ++++++++++-- weed/server/filer_server_handlers_write.go | 2 +- weed/shell/command_fs_configure.go | 26 ++++---- 10 files changed, 171 insertions(+), 57 deletions(-) diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index a327bdb17c..93e7e3ed32 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -771,7 +771,8 @@ message FilerConf { string replication = 3; string ttl = 4; string disk_type = 5; - bool fsync = 6; + // unset inherits from the enclosing path rule, set overrides it + optional bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; string data_center = 9; diff --git a/weed/filer/filer.go b/weed/filer/filer.go index 5a0090e61b..59e9bfc714 100644 --- a/weed/filer/filer.go +++ b/weed/filer/filer.go @@ -434,7 +434,7 @@ func SyncWrite(ctx context.Context) bool { // syncWriteContext flags ctx when the storage rule on p asks for fsync, so the // store write is held to the same durability as the data path. func (f *Filer) syncWriteContext(ctx context.Context, p util.FullPath) context.Context { - if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).Fsync { + if f.FilerConf != nil && f.FilerConf.MatchStorageRule(string(p)).GetFsync() { return WithSyncWrite(ctx) } return ctx diff --git a/weed/filer/filer_conf.go b/weed/filer/filer_conf.go index 55b8d8fc4d..e38235be41 100644 --- a/weed/filer/filer_conf.go +++ b/weed/filer/filer_conf.go @@ -33,10 +33,11 @@ const ( ) // FilerConfVersion is stamped into every configuration this build writes. -// Version 0 predates worm presence: it was written with EmitUnpopulated, so every -// rule carries an explicit "worm": false that meant nothing. Reading one back as an -// override would silently lift worm off nested paths, so it is dropped to unset. -const FilerConfVersion = 1 +// Version 0 predates worm presence and version 1 predates fsync presence: both +// were written with EmitUnpopulated, so every rule carries an explicit false +// that meant nothing. Reading one back as an override would silently lift the +// setting off nested paths, so a false from an older version is dropped to unset. +const FilerConfVersion = 2 type FilerConf struct { rules ptrie.Trie[*filer_pb.FilerConf_PathConf] @@ -126,9 +127,12 @@ func (fc *FilerConf) LoadFromBytes(data []byte) (err error) { func (fc *FilerConf) doLoadConf(conf *filer_pb.FilerConf) (err error) { for _, location := range conf.Locations { - if conf.Version < FilerConfVersion && location.Worm != nil && !*location.Worm { + if conf.Version < 1 && location.Worm != nil && !*location.Worm { location.Worm = nil } + if conf.Version < 2 && location.Fsync != nil && !*location.Fsync { + location.Fsync = nil + } err = fc.SetLocationConf(location) if err != nil { // this is not recoverable @@ -227,7 +231,10 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon if src == nil { return &filer_pb.FilerConf_PathConf{} } - var worm *bool + var fsync, worm *bool + if src.Fsync != nil { + fsync = proto.Bool(*src.Fsync) + } if src.Worm != nil { worm = proto.Bool(*src.Worm) } @@ -237,7 +244,7 @@ func ClonePathConf(src *filer_pb.FilerConf_PathConf) *filer_pb.FilerConf_PathCon Replication: src.Replication, Ttl: src.Ttl, DiskType: src.DiskType, - Fsync: src.Fsync, + Fsync: fsync, VolumeGrowthCount: src.VolumeGrowthCount, ReadOnly: src.ReadOnly, MaxFileNameLength: src.MaxFileNameLength, @@ -484,7 +491,7 @@ func ClearBucketLifecycleDayTTLs(ctx context.Context, client filer_pb.SeaweedFil // flag, a placement pin - is an operator's, and deleting the whole rule to // retire its TTL would take their configuration with it. func isLifecycleOwnedPathConf(c *filer_pb.FilerConf_PathConf) bool { - return c.GetDiskType() == "" && !c.GetFsync() && !c.GetReadOnly() && + return c.GetDiskType() == "" && c.Fsync == nil && !c.GetReadOnly() && c.GetDataCenter() == "" && c.GetRack() == "" && c.GetDataNode() == "" && c.GetMaxFileNameLength() == 0 && !c.GetDisableChunkDeletion() && c.Worm == nil && c.GetWormGracePeriodSeconds() == 0 && c.GetWormRetentionTimeSeconds() == 0 @@ -507,7 +514,12 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) { a.Replication = util.Nvl(b.Replication, a.Replication) a.Ttl = util.Nvl(b.Ttl, a.Ttl) a.DiskType = util.Nvl(b.DiskType, a.DiskType) - a.Fsync = b.Fsync || a.Fsync + // fsync and worm merge on presence, so a nested rule can turn them off. + // readOnly and disableChunkDeletion stay OR'ed on purpose: a nested rule + // must not be able to lift a lock the bucket set. + if b.Fsync != nil { + a.Fsync = proto.Bool(*b.Fsync) + } if b.VolumeGrowthCount > 0 { a.VolumeGrowthCount = b.VolumeGrowthCount } @@ -519,9 +531,6 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) { a.Rack = util.Nvl(b.Rack, a.Rack) a.DataNode = util.Nvl(b.DataNode, a.DataNode) a.DisableChunkDeletion = b.DisableChunkDeletion || a.DisableChunkDeletion - // worm merges on presence, so a nested rule can turn it off. readOnly, fsync and - // disableChunkDeletion stay OR'ed on purpose: a nested rule must not be able to - // lift a lock the bucket set. if b.Worm != nil { // copy the value: a is often a scratch conf while b is a live trie entry a.Worm = proto.Bool(*b.Worm) diff --git a/weed/filer/filer_conf_test.go b/weed/filer/filer_conf_test.go index b71da89b68..24f2ca591b 100644 --- a/weed/filer/filer_conf_test.go +++ b/weed/filer/filer_conf_test.go @@ -102,6 +102,54 @@ func TestWormLegacyFalseIsNotAnOverride(t *testing.T) { assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetWorm()) } +func TestFsyncInheritance(t *testing.T) { + fc := NewFilerConf() + fc.doLoadConf(&filer_pb.FilerConf{ + Version: FilerConfVersion, + Locations: []*filer_pb.FilerConf_PathConf{ + {LocationPrefix: "/buckets/b/", Fsync: proto.Bool(true), Ttl: "7d"}, + {LocationPrefix: "/buckets/b/quiet/", Collection: "quiet"}, + {LocationPrefix: "/buckets/b/scratch/", Fsync: proto.Bool(false)}, + {LocationPrefix: "/buckets/b/scratch/keep/", Fsync: proto.Bool(true)}, + }, + }) + + // a rule that says nothing about fsync keeps inheriting it, along with the ttl + rule := fc.MatchStorageRule("/buckets/b/quiet/x") + assert.True(t, rule.GetFsync()) + assert.Equal(t, "7d", rule.Ttl) + + // an explicit false turns it off, and a deeper rule turns it back on + assert.False(t, fc.MatchStorageRule("/buckets/b/scratch/x").GetFsync()) + assert.True(t, fc.MatchStorageRule("/buckets/b/scratch/keep/x").GetFsync()) + + // paths with no rule at all are unaffected + assert.False(t, fc.MatchStorageRule("/buckets/other/x").GetFsync()) +} + +// TestFsyncLegacyFalseIsNotAnOverride pins that the explicit "fsync": false +// every pre-version-2 configuration carries does not read as a per-path +// opt-out. +func TestFsyncLegacyFalseIsNotAnOverride(t *testing.T) { + const conf = `{ + "locations": [ + {"locationPrefix": "/buckets/b/", "fsync": true}, + {"locationPrefix": "/buckets/b/sub/", "collection": "sub", "fsync": false} + ] + }` + + for _, version := range []string{"", `"version": 0,`, `"version": 1,`} { + fc := NewFilerConf() + assert.NoError(t, fc.LoadFromBytes([]byte(`{`+version+conf[1:]))) + assert.True(t, fc.MatchStorageRule("/buckets/b/sub/x").GetFsync(), version) + } + + // the same file at the current version means what it says + fc := NewFilerConf() + assert.NoError(t, fc.LoadFromBytes([]byte(`{"version": 2,`+conf[1:]))) + assert.False(t, fc.MatchStorageRule("/buckets/b/sub/x").GetFsync()) +} + // TestWormSurvivesRoundTrip guards the write side: an explicit false has to be // stamped along with a version that says to honor it. func TestWormSurvivesRoundTrip(t *testing.T) { @@ -119,6 +167,23 @@ func TestWormSurvivesRoundTrip(t *testing.T) { assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetWorm()) } +// TestFsyncSurvivesRoundTrip guards the write side: an explicit false has to be +// stamped along with a version that says to honor it. +func TestFsyncSurvivesRoundTrip(t *testing.T) { + fc := NewFilerConf() + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/", Fsync: proto.Bool(true)}) + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/sub/", Fsync: proto.Bool(false)}) + fc.SetLocationConf(&filer_pb.FilerConf_PathConf{LocationPrefix: "/buckets/b/other/", Ttl: "7d"}) + + var buf bytes.Buffer + assert.NoError(t, fc.ToText(&buf)) + + reloaded := NewFilerConf() + assert.NoError(t, reloaded.LoadFromBytes(buf.Bytes())) + assert.False(t, reloaded.MatchStorageRule("/buckets/b/sub/x").GetFsync()) + assert.True(t, reloaded.MatchStorageRule("/buckets/b/other/x").GetFsync()) +} + // TestClonePathConf verifies that ClonePathConf copies all exported fields. // Uses reflection to automatically detect new fields added to the protobuf, // ensuring the test fails if ClonePathConf is not updated for new fields. @@ -130,7 +195,7 @@ func TestClonePathConf(t *testing.T) { Replication: "001", Ttl: "7d", DiskType: "ssd", - Fsync: true, + Fsync: proto.Bool(true), VolumeGrowthCount: 5, ReadOnly: true, MaxFileNameLength: 255, @@ -189,9 +254,11 @@ func TestClonePathConf(t *testing.T) { // Verify mutation of clone doesn't affect source clone.Collection = "modified" clone.ReadOnly = false + *clone.Fsync = false *clone.Worm = false assert.Equal(t, "test_collection", src.Collection, "Modifying clone should not affect source Collection") assert.Equal(t, true, src.ReadOnly, "Modifying clone should not affect source ReadOnly") + assert.Equal(t, true, src.GetFsync(), "Modifying clone should not affect source Fsync") assert.Equal(t, true, src.GetWorm(), "Modifying clone should not affect source Worm") } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index dca77d2eca..1919108fcd 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -786,7 +786,8 @@ message FilerConf { string replication = 3; string ttl = 4; string disk_type = 5; - bool fsync = 6; + // unset inherits from the enclosing path rule, set overrides it + optional bool fsync = 6; uint32 volume_growth_count = 7; bool read_only = 8; string data_center = 9; diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 94a21acc92..f00ed17771 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -7102,20 +7102,21 @@ func (x *LocateBrokerResponse_Resource) GetResourceCount() int32 { } type FilerConf_PathConf struct { - state protoimpl.MessageState `protogen:"open.v1"` - LocationPrefix string `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"` - Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` - Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"` - Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"` - DiskType string `protobuf:"bytes,5,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` - Fsync bool `protobuf:"varint,6,opt,name=fsync,proto3" json:"fsync,omitempty"` - VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"` - ReadOnly bool `protobuf:"varint,8,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"` - DataCenter string `protobuf:"bytes,9,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` - Rack string `protobuf:"bytes,10,opt,name=rack,proto3" json:"rack,omitempty"` - DataNode string `protobuf:"bytes,11,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` - MaxFileNameLength uint32 `protobuf:"varint,12,opt,name=max_file_name_length,json=maxFileNameLength,proto3" json:"max_file_name_length,omitempty"` - DisableChunkDeletion bool `protobuf:"varint,13,opt,name=disable_chunk_deletion,json=disableChunkDeletion,proto3" json:"disable_chunk_deletion,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + LocationPrefix string `protobuf:"bytes,1,opt,name=location_prefix,json=locationPrefix,proto3" json:"location_prefix,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection,proto3" json:"collection,omitempty"` + Replication string `protobuf:"bytes,3,opt,name=replication,proto3" json:"replication,omitempty"` + Ttl string `protobuf:"bytes,4,opt,name=ttl,proto3" json:"ttl,omitempty"` + DiskType string `protobuf:"bytes,5,opt,name=disk_type,json=diskType,proto3" json:"disk_type,omitempty"` + // unset inherits from the enclosing path rule, set overrides it + Fsync *bool `protobuf:"varint,6,opt,name=fsync,proto3,oneof" json:"fsync,omitempty"` + VolumeGrowthCount uint32 `protobuf:"varint,7,opt,name=volume_growth_count,json=volumeGrowthCount,proto3" json:"volume_growth_count,omitempty"` + ReadOnly bool `protobuf:"varint,8,opt,name=read_only,json=readOnly,proto3" json:"read_only,omitempty"` + DataCenter string `protobuf:"bytes,9,opt,name=data_center,json=dataCenter,proto3" json:"data_center,omitempty"` + Rack string `protobuf:"bytes,10,opt,name=rack,proto3" json:"rack,omitempty"` + DataNode string `protobuf:"bytes,11,opt,name=data_node,json=dataNode,proto3" json:"data_node,omitempty"` + MaxFileNameLength uint32 `protobuf:"varint,12,opt,name=max_file_name_length,json=maxFileNameLength,proto3" json:"max_file_name_length,omitempty"` + DisableChunkDeletion bool `protobuf:"varint,13,opt,name=disable_chunk_deletion,json=disableChunkDeletion,proto3" json:"disable_chunk_deletion,omitempty"` // unset inherits from the enclosing path rule, set overrides it Worm *bool `protobuf:"varint,14,opt,name=worm,proto3,oneof" json:"worm,omitempty"` WormGracePeriodSeconds uint64 `protobuf:"varint,15,opt,name=worm_grace_period_seconds,json=wormGracePeriodSeconds,proto3" json:"worm_grace_period_seconds,omitempty"` @@ -7190,8 +7191,8 @@ func (x *FilerConf_PathConf) GetDiskType() string { } func (x *FilerConf_PathConf) GetFsync() bool { - if x != nil { - return x.Fsync + if x != nil && x.Fsync != nil { + return *x.Fsync } return false } @@ -7764,10 +7765,10 @@ const file_filer_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\fR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\fR\x05value\"%\n" + "\rKvPutResponse\x12\x14\n" + - "\x05error\x18\x01 \x01(\tR\x05error\"\xc0\x05\n" + + "\x05error\x18\x01 \x01(\tR\x05error\"\xcf\x05\n" + "\tFilerConf\x12\x18\n" + "\aversion\x18\x01 \x01(\x05R\aversion\x12:\n" + - "\tlocations\x18\x02 \x03(\v2\x1c.filer_pb.FilerConf.PathConfR\tlocations\x1a\xdc\x04\n" + + "\tlocations\x18\x02 \x03(\v2\x1c.filer_pb.FilerConf.PathConfR\tlocations\x1a\xeb\x04\n" + "\bPathConf\x12'\n" + "\x0flocation_prefix\x18\x01 \x01(\tR\x0elocationPrefix\x12\x1e\n" + "\n" + @@ -7775,8 +7776,8 @@ const file_filer_proto_rawDesc = "" + "collection\x12 \n" + "\vreplication\x18\x03 \x01(\tR\vreplication\x12\x10\n" + "\x03ttl\x18\x04 \x01(\tR\x03ttl\x12\x1b\n" + - "\tdisk_type\x18\x05 \x01(\tR\bdiskType\x12\x14\n" + - "\x05fsync\x18\x06 \x01(\bR\x05fsync\x12.\n" + + "\tdisk_type\x18\x05 \x01(\tR\bdiskType\x12\x19\n" + + "\x05fsync\x18\x06 \x01(\bH\x00R\x05fsync\x88\x01\x01\x12.\n" + "\x13volume_growth_count\x18\a \x01(\rR\x11volumeGrowthCount\x12\x1b\n" + "\tread_only\x18\b \x01(\bR\breadOnly\x12\x1f\n" + "\vdata_center\x18\t \x01(\tR\n" + @@ -7786,9 +7787,10 @@ const file_filer_proto_rawDesc = "" + "\tdata_node\x18\v \x01(\tR\bdataNode\x12/\n" + "\x14max_file_name_length\x18\f \x01(\rR\x11maxFileNameLength\x124\n" + "\x16disable_chunk_deletion\x18\r \x01(\bR\x14disableChunkDeletion\x12\x17\n" + - "\x04worm\x18\x0e \x01(\bH\x00R\x04worm\x88\x01\x01\x129\n" + + "\x04worm\x18\x0e \x01(\bH\x01R\x04worm\x88\x01\x01\x129\n" + "\x19worm_grace_period_seconds\x18\x0f \x01(\x04R\x16wormGracePeriodSeconds\x12=\n" + - "\x1bworm_retention_time_seconds\x18\x10 \x01(\x04R\x18wormRetentionTimeSecondsB\a\n" + + "\x1bworm_retention_time_seconds\x18\x10 \x01(\x04R\x18wormRetentionTimeSecondsB\b\n" + + "\x06_fsyncB\a\n" + "\x05_worm\"\xba\x01\n" + "&CacheRemoteObjectToLocalClusterRequest\x12\x1c\n" + "\tdirectory\x18\x01 \x01(\tR\tdirectory\x12\x12\n" + diff --git a/weed/pb/filer_pb/filer_grpc.pb.go b/weed/pb/filer_pb/filer_grpc.pb.go index 8a330ce286..6a2b494a39 100644 --- a/weed/pb/filer_pb/filer_grpc.pb.go +++ b/weed/pb/filer_pb/filer_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.6.2 -// - protoc v3.21.12 +// - protoc v7.35.0 // source: filer.proto package filer_pb diff --git a/weed/pb/filer_pb/filer_vtproto.pb.go b/weed/pb/filer_pb/filer_vtproto.pb.go index cac2d01f9a..e86380465e 100644 --- a/weed/pb/filer_pb/filer_vtproto.pb.go +++ b/weed/pb/filer_pb/filer_vtproto.pb.go @@ -2659,6 +2659,16 @@ func (m *AtomicRenameEntryRequest) MarshalToSizedBufferVT(dAtA []byte) (int, err i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.FailIfTargetExists { + i-- + if m.FailIfTargetExists { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x30 + } if len(m.Signatures) > 0 { var pksize2 int for _, num := range m.Signatures { @@ -4984,9 +4994,9 @@ func (m *FilerConf_PathConf) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0x38 } - if m.Fsync { + if m.Fsync != nil { i-- - if m.Fsync { + if *m.Fsync { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -7599,6 +7609,9 @@ func (m *AtomicRenameEntryRequest) SizeVT() (n int) { } n += 1 + protohelpers.SizeOfVarint(uint64(l)) + l } + if m.FailIfTargetExists { + n += 2 + } n += len(m.unknownFields) return n } @@ -8501,7 +8514,7 @@ func (m *FilerConf_PathConf) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } - if m.Fsync { + if m.Fsync != nil { n += 2 } if m.VolumeGrowthCount != 0 { @@ -16531,6 +16544,26 @@ func (m *AtomicRenameEntryRequest) UnmarshalVT(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field Signatures", wireType) } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FailIfTargetExists", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.FailIfTargetExists = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -22263,7 +22296,8 @@ func (m *FilerConf_PathConf) UnmarshalVT(dAtA []byte) error { break } } - m.Fsync = bool(v != 0) + b := bool(v != 0) + m.Fsync = &b case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field VolumeGrowthCount", wireType) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 177093c501..9009a0f045 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -337,7 +337,7 @@ func (fs *FilerServer) detectStorageOption(ctx context.Context, requestURI, qCol DataNode: util.Nvl(dataNode, rule.DataNode, fs.option.DataNode), TtlSeconds: ttlSeconds, DiskType: util.Nvl(diskType, overlayDisk, rule.DiskType), - Fsync: rule.Fsync, + Fsync: rule.GetFsync(), VolumeGrowthCount: rule.VolumeGrowthCount, MaxFileNameLength: maxFileNameLength, // A location that survives from a request parameter, a placement diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 29be9b87ae..4bc29e3984 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -67,7 +67,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io replication := fsConfigureCommand.String("replication", "", "assign writes with this replication") ttl := fsConfigureCommand.String("ttl", "", "assign writes with this ttl (e.g., 1m, 1h, 1d, 1w, 1y)") diskType := fsConfigureCommand.String("disk", "", "[hdd|ssd|] hard drive or solid state drive or any tag") - fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes") + fsync := fsConfigureCommand.Bool("fsync", false, "fsync for the writes; unset inherits from the parent path") isReadOnly := fsConfigureCommand.Bool("readOnly", false, "disable writes") worm := fsConfigureCommand.Bool("worm", false, "write-once-read-many, written files are readonly; unset inherits from the parent path") wormGracePeriod := fsConfigureCommand.Uint64("wormGracePeriod", 0, "grace period before worm is enforced, in seconds") @@ -101,7 +101,6 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io Collection: *collection, Replication: *replication, Ttl: *ttl, - Fsync: *fsync, MaxFileNameLength: uint32(*maxFileNameLength), DiskType: *diskType, VolumeGrowthCount: uint32(*volumeGrowthCount), @@ -113,10 +112,13 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io WormRetentionTimeSeconds: *wormRetentionTime, } - // worm is only carried when the flag is passed, so a rule that says nothing - // about it keeps inheriting from the enclosing path + // fsync and worm are only carried when the flag is passed, so a rule that + // says nothing about them keeps inheriting from the enclosing path fsConfigureCommand.Visit(func(f *flag.Flag) { - if f.Name == "worm" { + switch f.Name { + case "fsync": + locConf.Fsync = proto.Bool(*fsync) + case "worm": locConf.Worm = proto.Bool(*worm) } }) @@ -152,17 +154,15 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io fc.DeleteLocationConf(*locationPrefix) } else { fc.AddLocationConf(locConf) - // AddLocationConf merges these boolean fields with OR, which can never - // turn a flag off; let an explicitly passed false win, e.g. -readOnly=false - // to reopen a bucket that quota enforcement locked. worm does not belong - // here: it merges on presence, so the value set above already wins. + // AddLocationConf merges this boolean field with OR, which can never + // turn it off; let an explicitly passed false win, e.g. -readOnly=false + // to reopen a bucket that quota enforcement locked. fsync and worm do + // not belong here: they merge on presence, so the values set above + // already win. if mergedConf, found := fc.GetLocationConf(*locationPrefix); found { fsConfigureCommand.Visit(func(f *flag.Flag) { - switch f.Name { - case "readOnly": + if f.Name == "readOnly" { mergedConf.ReadOnly = *isReadOnly - case "fsync": - mergedConf.Fsync = *fsync } }) } From 0d00e921789731777281f765eb6790eb3349f5e4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 20:49:49 -0700 Subject: [PATCH 38/70] iceberg maintenance: match percent-decoded keys when checking file references (#11978) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * iceberg maintenance: match percent-decoded keys when checking file references Writers may percent-escape partition directory names in the file paths they record in manifests (e.g. "part%3D1"), while the S3 layer decodes the key before storing the object. remove_orphans compared the escaped reference to the decoded filer path, counted the live file as an orphan, and deleted it once it passed the age window — readers then got NoSuchKey on the next scan. Match both spellings of every referenced path in remove_orphans, apply the same coverage to the kept-file set in expire_snapshots, and delete expired files at their decoded key so the object the S3 layer stored is the one removed. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * iceberg maintenance: retry filer lookups at the percent-decoded key A manifest reference carrying an escaped segment resolves to a filer name that does not exist, because the S3 layer decoded the key on write. When a lookup misses and the reference contains escapes, retry once at the decoded spelling so maintenance can still read the file. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * iceberg maintenance: resolve referenced files verbatim-first, decoded fallback filer_pb.LookupEntry reports misses as ErrNotFound rather than a gRPC NotFound, so the escaped-reference retry never fired against a real filer. Both maintenance reads and expired-snapshot deletes now resolve through one verbatim-first/decoded-fallback lookup, and expiry refuses to delete a resolved path outside the table's data directory. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * iceberg maintenance: test expiry of a file stored at a literal percent key * iceberg maintenance: resolve referenced files in one filer lookup lookupFileByIcebergPath re-fetched the entry resolveFilerPath had just looked up, doubling the RPC per referenced file during compaction and expiration. The resolver now returns the entry alongside the path so a hit costs one lookup and a decoded fallback two. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * iceberg maintenance: keep every decoded spelling of a referenced path The kept-reference sets recorded a path plus a single decode of it, but lookupResolvedEntry decodes recursively until the spelling is stable. A doubly-escaped reference could resolve to a spelling absent from the set and be deleted while still referenced. addFilerPathSpellings now walks the same decode chain the resolver does. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * iceberg maintenance: keep every decoded spelling of a referenced path * iceberg maintenance: test expiry keeps a doubly-escaped kept reference * iceberg maintenance: clean decoded spellings in the referenced-path set * iceberg maintenance: test expiry keeps a kept reference resolved via decoded parent segment * iceberg maintenance: decode only the reference in the spelling set Decoding the composed filer path also decoded the table directory, so a table at a literal percent-escaped path produced a kept set missing the spelling the resolver returns, and expiry could delete a live file. * test: gofmt the s3tables cluster helpers Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/worker/tasks/iceberg/exec_test.go | 435 ++++++++++++++++++++++++ weed/worker/tasks/iceberg/filer_io.go | 75 +++- weed/worker/tasks/iceberg/operations.go | 24 +- 3 files changed, 509 insertions(+), 25 deletions(-) diff --git a/weed/worker/tasks/iceberg/exec_test.go b/weed/worker/tasks/iceberg/exec_test.go index 18cb661ca6..172e896a1e 100644 --- a/weed/worker/tasks/iceberg/exec_test.go +++ b/weed/worker/tasks/iceberg/exec_test.go @@ -4435,3 +4435,438 @@ func TestResolveCompactionRewritePlanFallsBackForUnsupportedSortTransform(t *tes t.Fatalf("expected fallback to %q for unsupported transform, got %+v", defaultRewriteStrategy, plan) } } + +// A writer may percent-escape a partition directory name ("part=1" becomes +// "part%3D1") in the file paths it records in manifests, while the S3 layer +// decodes the key before storing it. Orphan cleanup must match the decoded +// spelling or it deletes live data files once they pass the age window. +func TestRemoveOrphansKeepsPercentEscapedReferences(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now().UnixMilli() + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now, ManifestList: "metadata/snap-1.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + oldTime := time.Now().Add(-200 * time.Hour).Unix() + + // Rewrite the manifest so its entries carry the escaped spelling the + // writer recorded. + escapedPath := fmt.Sprintf("s3://%s/%s/data/month_collected_at_1%%3D681/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), + iceberg.EntryContentData, + escapedPath, + iceberg.ParquetFile, + map[int]any{}, + nil, nil, + 10, + 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime, FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + // The object lives at the decoded key, the same place the S3 PUT landed it. + fs.putEntry(dataDir, "month_collected_at_1=681", &filer_pb.Entry{ + Name: "month_collected_at_1=681", + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime}, + }) + partDir := path.Join(dataDir, "month_collected_at_1=681") + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", + Attributes: &filer_pb.FuseAttributes{Mtime: oldTime, FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{OrphanOlderThanHours: 0, MaxCommitRetries: 3} + + result, _, err := handler.removeOrphans(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("removeOrphans failed: %v", err) + } + if !strings.Contains(result, "removed 1 orphan") { + t.Errorf("expected only the unreferenced snap-1-data.parquet removed, got %q", result) + } + if fs.getEntry(partDir, "f.parquet") == nil { + t.Error("data file referenced through an escaped manifest path was deleted") + } +} + +// The sibling of the orphan-sweep case on the read side: a manifest reference +// carrying a percent-escaped segment must find the object the S3 layer stored +// at the decoded key. +func TestLookupFileByIcebergPathDecodesEscapedReference(t *testing.T) { + fs, client := startFakeFiler(t) + + dir := path.Join(s3tables.TablesPath, "bkt", "ns/tbl/data/part=1") + fs.putEntry(dir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", + Content: []byte("data"), + }) + literalDir := path.Join(s3tables.TablesPath, "bkt", "ns/tbl/data/part%3D2") + fs.putEntry(literalDir, "g.parquet", &filer_pb.Entry{ + Name: "g.parquet", + Content: []byte("data"), + }) + + entry, err := lookupFileByIcebergPath(context.Background(), client, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D1/f.parquet") + if err != nil { + t.Fatalf("escaped reference should resolve to the decoded key: %v", err) + } + if entry == nil || entry.Name != "f.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } + + // A file genuinely stored under an escaped name still resolves verbatim. + entry, err = lookupFileByIcebergPath(context.Background(), client, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D2/g.parquet") + if err != nil { + t.Fatalf("literal escaped key should resolve: %v", err) + } + if entry == nil || entry.Name != "g.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } + + // A real filer reports misses as filer_pb.ErrNotFound, not a gRPC NotFound + // status; the decoded retry must trigger on that shape too. + entry, err = lookupFileByIcebergPath(context.Background(), errNotFoundClient{client}, "bkt", "ns/tbl", "s3://bkt/ns/tbl/data/part%3D1/f.parquet") + if err != nil { + t.Fatalf("escaped reference should resolve under ErrNotFound misses: %v", err) + } + if entry == nil || entry.Name != "f.parquet" { + t.Fatalf("resolved to wrong entry: %+v", entry) + } +} + +// errNotFoundClient converts gRPC NotFound misses into filer_pb.ErrNotFound, +// the shape a real filer produces through filer_pb.LookupEntry. +type errNotFoundClient struct { + filer_pb.SeaweedFilerClient +} + +func (c errNotFoundClient) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { + resp, err := c.SeaweedFilerClient.LookupDirectoryEntry(ctx, req, opts...) + if status.Code(err) == codes.NotFound { + return nil, filer_pb.ErrNotFound + } + return resp, err +} + +func TestExpireSnapshotsDeletesEscapedReferenceAtExistingKey(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + // Snapshot 1 (expired) references a file the S3 layer stored at the decoded + // key; snapshot 2 (kept) does not reference it. + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%3D1/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + partDir := path.Join(dataDir, "month=1") + fs.putEntry(dataDir, "month=1", &filer_pb.Entry{ + Name: "month=1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + result, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + t.Logf("expireSnapshots result: %s", result) + + if fs.getEntry(partDir, "f.parquet") != nil { + t.Error("expired file at the decoded key should have been deleted") + } +} + +// A client that escapes its own percent sign on upload leaves an object under +// a literal %3D-containing key while the manifest spells the same reference. +// Expiration must delete the entry that actually exists, not the decoded +// spelling. +func TestExpireSnapshotsDeletesLiteralPercentKey(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%3D1/f.parquet", setup.BucketName, setup.tablePath()) + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + snapID := int64(1) + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", "manifest-1.avro"), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, "manifest-1.avro", &filer_pb.Entry{ + Name: "manifest-1.avro", + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + + literalDir := path.Join(dataDir, "month%3D1") + fs.putEntry(dataDir, "month%3D1", &filer_pb.Entry{ + Name: "month%3D1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(literalDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + result, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config) + if err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + t.Logf("expireSnapshots result: %s", result) + + if fs.getEntry(literalDir, "f.parquet") != nil { + t.Error("expired file at the literal percent key should have been deleted") + } +} + +// A client that escapes an already-escaped name writes a manifest reference +// like month%253D1 while the object sits at month=1. The kept set must hold +// every decoding in the chain, or the expiry pass deletes a file a live +// snapshot still references. +func TestExpireSnapshotsKeepsDoublyEscapedKeptReference(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + escapedPath := fmt.Sprintf("s3://%s/%s/data/month%%253D1/f.parquet", setup.BucketName, setup.tablePath()) + writeRefManifest := func(fileName string, snapID int64) { + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, escapedPath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build escaped data file: %v", err) + } + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", fileName), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, fileName, &filer_pb.Entry{ + Name: fileName, + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + } + writeRefManifest("manifest-1.avro", 1) + writeRefManifest("manifest-2.avro", 2) + + partDir := path.Join(dataDir, "month=1") + fs.putEntry(dataDir, "month=1", &filer_pb.Entry{ + Name: "month=1", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix()}, + }) + fs.putEntry(partDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + if _, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config); err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + + if fs.getEntry(partDir, "f.parquet") == nil { + t.Error("file referenced by the kept snapshot through a doubly-escaped path was deleted") + } +} + +// A kept reference whose decoded spelling contains a parent segment resolves +// to the cleaned path. The kept set must record the cleaned spelling, or the +// expiry pass misses the protection and deletes a file a live snapshot still +// references. +func TestExpireSnapshotsKeepsDecodedParentReference(t *testing.T) { + fs, client := startFakeFiler(t) + + now := time.Now() + old := now.Add(-10 * 24 * time.Hour) + setup := tableSetup{ + BucketName: "test-bucket", + Namespace: "ns", + TableName: "tbl", + Age: 10 * 24 * time.Hour, + Snapshots: []table.Snapshot{ + {SnapshotID: 1, TimestampMs: now.UnixMilli(), ManifestList: "metadata/snap-1.avro"}, + {SnapshotID: 2, TimestampMs: now.UnixMilli() + 1, ManifestList: "metadata/snap-2.avro"}, + }, + } + meta := populateTable(t, fs, setup) + + metaDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "metadata") + dataDir := path.Join(s3tables.TablesPath, setup.BucketName, setup.tablePath(), "data") + + writeRefManifest := func(fileName string, snapID int64, filePath string) { + dfBuilder, err := iceberg.NewDataFileBuilder( + meta.PartitionSpec(), iceberg.EntryContentData, filePath, + iceberg.ParquetFile, map[int]any{}, nil, nil, 10, 4096, + ) + if err != nil { + t.Fatalf("build data file: %v", err) + } + var manifestBuf bytes.Buffer + if _, err := iceberg.WriteManifest( + setup.fileRef("metadata", fileName), &manifestBuf, + meta.Version(), meta.PartitionSpec(), meta.CurrentSchema(), snapID, + []iceberg.ManifestEntry{iceberg.NewManifestEntry(iceberg.EntryStatusADDED, &snapID, nil, nil, dfBuilder.Build())}, + ); err != nil { + t.Fatalf("write manifest: %v", err) + } + fs.putEntry(metaDir, fileName, &filer_pb.Entry{ + Name: fileName, + Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: uint64(manifestBuf.Len())}, + Content: manifestBuf.Bytes(), + }) + } + writeRefManifest("manifest-1.avro", 1, fmt.Sprintf("s3://%s/%s/data/f.parquet", setup.BucketName, setup.tablePath())) + writeRefManifest("manifest-2.avro", 2, fmt.Sprintf("s3://%s/%s/data/tmp/%%2e%%2e/f.parquet", setup.BucketName, setup.tablePath())) + + fs.putEntry(dataDir, "f.parquet", &filer_pb.Entry{ + Name: "f.parquet", Attributes: &filer_pb.FuseAttributes{Mtime: old.Unix(), FileSize: 4096}, + Content: []byte("fake-parquet-data"), + }) + + handler := NewHandler(nil) + config := Config{ + SnapshotRetentionMs: hoursToMs(0), + MaxSnapshotsToKeep: 1, + MaxCommitRetries: 3, + Operations: "expire_snapshots", + } + + if _, _, err := handler.expireSnapshots(context.Background(), client, setup.BucketName, setup.tablePath(), config); err != nil { + t.Fatalf("expireSnapshots failed: %v", err) + } + + if fs.getEntry(dataDir, "f.parquet") == nil { + t.Error("file referenced by the kept snapshot through a decoded parent segment was deleted") + } +} diff --git a/weed/worker/tasks/iceberg/filer_io.go b/weed/worker/tasks/iceberg/filer_io.go index 482c8dc80a..9c05e010d2 100644 --- a/weed/worker/tasks/iceberg/filer_io.go +++ b/weed/worker/tasks/iceberg/filer_io.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "net/url" "path" "strconv" "strings" @@ -162,26 +163,66 @@ func loadCurrentMetadata(ctx context.Context, client filer_pb.SeaweedFilerClient return state, nil } +func isFilerNotFound(err error) bool { + return errors.Is(err, filer_pb.ErrNotFound) || status.Code(err) == codes.NotFound +} + +// resolveFilerPath returns the filer path that actually holds a referenced +// file: the verbatim spelling first, then the percent-decoded one. Writers +// may record a path with escaped segments while the S3 layer decoded the key +// before storing the object. +func resolveFilerPath(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (string, error) { + _, fullPath, err := lookupResolvedEntry(ctx, client, bucketName, dataPath, icebergPath) + return fullPath, err +} + +// lookupResolvedEntry returns the entry and filer path holding an +// Iceberg-style reference under the verbatim-first rule. +func lookupResolvedEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (*filer_pb.Entry, string, error) { + fullPath, err := icebergFilerPath(bucketName, dataPath, icebergPath) + if err != nil { + return nil, "", err + } + resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{ + Directory: path.Dir(fullPath), + Name: path.Base(fullPath), + }) + if err == nil { + return resp.Entry, fullPath, nil + } + if !isFilerNotFound(err) { + return nil, "", err + } + if decoded, decErr := url.PathUnescape(icebergPath); decErr == nil && decoded != icebergPath { + return lookupResolvedEntry(ctx, client, bucketName, dataPath, decoded) + } + return nil, "", err +} + +// addFilerPathSpellings records every filer path lookupResolvedEntry can +// return for a reference: each successive decoding of the reference, with the +// rest of the path left verbatim. An extra spelling only keeps a file; a +// missing one can drop a referenced file. +func addFilerPathSpellings(set map[string]struct{}, bucketName, dataPath, icebergPath string) { + for spelling := icebergPath; ; { + filerPath, err := icebergFilerPath(bucketName, dataPath, spelling) + if err != nil { + return + } + set[filerPath] = struct{}{} + decoded, err := url.PathUnescape(spelling) + if err != nil || decoded == spelling { + return + } + spelling = decoded + } +} + // lookupFileByIcebergPath returns the filer entry holding an Iceberg-style // path. dataPath is the bucket-relative directory the table's files live in. func lookupFileByIcebergPath(ctx context.Context, client filer_pb.SeaweedFilerClient, bucketName, dataPath, icebergPath string) (*filer_pb.Entry, error) { - fullPath, err := icebergFilerPath(bucketName, dataPath, icebergPath) - if err != nil { - return nil, err - } - dir, fileName := path.Dir(fullPath), path.Base(fullPath) - - resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{ - Directory: dir, - Name: fileName, - }) - if err != nil { - return nil, fmt.Errorf("lookup %s/%s: %w", dir, fileName, err) - } - if resp == nil || resp.Entry == nil { - return nil, fmt.Errorf("file not found: %s/%s", dir, fileName) - } - return resp.Entry, nil + entry, _, err := lookupResolvedEntry(ctx, client, bucketName, dataPath, icebergPath) + return entry, err } // loadFileByIcebergPath loads a file from the filer given an Iceberg-style path. diff --git a/weed/worker/tasks/iceberg/operations.go b/weed/worker/tasks/iceberg/operations.go index 2266ad0ae3..8b7bb22f24 100644 --- a/weed/worker/tasks/iceberg/operations.go +++ b/weed/worker/tasks/iceberg/operations.go @@ -9,6 +9,7 @@ import ( "math/rand/v2" "path" "sort" + "strings" "time" "github.com/apache/iceberg-go" @@ -90,9 +91,7 @@ func (h *Handler) expireSnapshots( // Resolve kept file paths for consistent comparison keptFilerPaths := make(map[string]struct{}, len(keptFiles)) for f := range keptFiles { - if filerPath, err := icebergFilerPath(bucketName, dataPath, f); err == nil { - keptFilerPaths[filerPath] = struct{}{} - } + addFilerPathSpellings(keptFilerPaths, bucketName, dataPath, f) } // Use MetadataBuilder to remove snapshots and create new metadata @@ -118,11 +117,18 @@ func (h *Handler) expireSnapshots( } // Delete files exclusively referenced by expired snapshots (best-effort) + tableBase := path.Join(s3tables.TablesPath, bucketName, dataPath) + "/" deletedCount := 0 for filePath := range expiredFiles { - resolved, err := icebergFilerPath(bucketName, dataPath, filePath) + resolved, err := resolveFilerPath(ctx, filerClient, bucketName, dataPath, filePath) if err != nil { - glog.Warningf("iceberg maintenance: cannot resolve expired file %s: %v", filePath, err) + if !isFilerNotFound(err) { + glog.Warningf("iceberg maintenance: cannot resolve expired file %s: %v", filePath, err) + } + continue + } + if !strings.HasPrefix(resolved, tableBase) { + glog.Warningf("iceberg maintenance: refusing to delete %s outside table path", resolved) continue } if _, stillReferenced := keptFilerPaths[resolved]; stillReferenced { @@ -357,9 +363,11 @@ func collectOrphanCandidates( referencedFilerPaths := make(map[string]struct{}, len(referencedFiles)) for ref := range referencedFiles { - if filerPath, err := icebergFilerPath(bucketName, dataPath, ref); err == nil { - referencedFilerPaths[filerPath] = struct{}{} - } + // Writers percent-escape partition directory names (e.g. "a=1" stored + // as "a%3D1" in the manifest path), but the S3 layer decodes the key + // before it lands on the filer. Match every spelling the resolver can + // return, or a live file is counted as an orphan and deleted. + addFilerPathSpellings(referencedFilerPaths, bucketName, dataPath, ref) } tableBasePath := path.Join(s3tables.TablesPath, bucketName, dataPath) From 81e0859c122c9a827eab9487aea163387c476b77 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 20:54:52 -0700 Subject: [PATCH 39/70] test(s3 remote cache): wait for the master to list the filer before shell commands (#12018) A fresh single-filer mini registers its filer only after the ~10s empty-membership confirm window, and weed shell locates the filer via the master's listing, so remote.configure sent right after the S3 health check races it. Ports the wait from the enterprise test to the OSS copy. --- .../remote_cache/remote_cache_evict_test.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/s3/remote_cache/remote_cache_evict_test.go b/test/s3/remote_cache/remote_cache_evict_test.go index cb44f1f3fb..8fab9c53b9 100644 --- a/test/s3/remote_cache/remote_cache_evict_test.go +++ b/test/s3/remote_cache/remote_cache_evict_test.go @@ -99,6 +99,27 @@ func shellOn(t *testing.T, masterPort, command string) string { return stripLogs(string(out)) } +// waitForFilerListed polls the master until it lists a filer. A fresh +// single-filer mini registers its filer only after an ~10 s empty-membership +// confirm, and weed shell finds the filer through that listing, so shell +// commands sent before it fail. +func waitForFilerListed(t *testing.T, masterPort string) { + t.Helper() + deadline := time.Now().Add(60 * time.Second) + for { + cmd := exec.Command(weedBinary, "shell", "-master=localhost:"+masterPort) + cmd.Stdin = strings.NewReader("cluster.ps\nexit\n") + out, err := cmd.CombinedOutput() + if err == nil && strings.Contains(string(out), "* filers ") && !strings.Contains(string(out), "* filers 0\n") { + return + } + if time.Now().After(deadline) { + t.Fatalf("master on %s lists no filer after 60s: %v\n%s", masterPort, err, out) + } + time.Sleep(time.Second) + } +} + func chunkCountOn(t *testing.T, masterPort, path string) string { meta := shellOn(t, masterPort, "fs.meta.cat "+path) idx := strings.LastIndex(meta, "chunks ") @@ -166,6 +187,7 @@ func TestRemoteCacheEvictUnderPressure(t *testing.T) { "-master.volumeSizeLimitMB=32", "-volume.max=2", "-filer.remoteCacheEvictThreshold=0.99") waitForHTTP(t, evictPrimaryS3) + waitForFilerListed(t, evictPrimaryMaster) remote := createS3Client(evictRemoteS3) _, err := remote.CreateBucket(&s3.CreateBucketInput{Bucket: aws.String(evictBucket)}) From aba0cb7b2a0d0e81d23b1e590fc44d5da6f84301 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 22:31:15 -0700 Subject: [PATCH 40/70] ci: isolate S3 lifecycle mini from LAN clusters (#12022) --- test/s3/lifecycle/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/s3/lifecycle/Makefile b/test/s3/lifecycle/Makefile index 98aec8eceb..cf55c424e6 100644 --- a/test/s3/lifecycle/Makefile +++ b/test/s3/lifecycle/Makefile @@ -42,6 +42,8 @@ start-server: build-weed @mkdir -p $(SERVER_DIR) @AWS_ACCESS_KEY_ID=$(ACCESS_KEY) AWS_SECRET_ACCESS_KEY=$(SECRET_KEY) $(WEED_BINARY) mini \ -dir=$(SERVER_DIR) \ + -ip=127.0.0.1 \ + -ip.bind=127.0.0.1 \ -s3.port=$(S3_PORT) \ -filer.port.grpc=$(FILER_GRPC_PORT) \ > weed-test.log 2>&1 & \ From a1b3131bf682a48f4b713799a66660ee6887cc7f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 22:40:54 -0700 Subject: [PATCH 41/70] test: handle unsupported catalog clients in S3 Tables CI (#12023) test: distinguish unsupported catalog clients from S3 Tables failures --- .../duckdb_deterministic_location_test.go | 3 ++ test/s3tables/catalog/duckdb_oauth_test.go | 6 +++ test/s3tables/catalog/iceberg_catalog_test.go | 3 ++ test/s3tables/catalog/issue_9103_test.go | 3 ++ test/s3tables/catalog/lance_client_test.go | 7 +++- .../catalog_lancedb/lancedb_catalog_test.go | 3 ++ .../s3tables/catalog_risingwave/setup_test.go | 23 +++++++++++ test/testutil/helpers.go | 5 +++ test/testutil/helpers_test.go | 39 +++++++++++++++++++ 9 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 test/testutil/helpers_test.go diff --git a/test/s3tables/catalog/duckdb_deterministic_location_test.go b/test/s3tables/catalog/duckdb_deterministic_location_test.go index 8cccdddadc..256efb68c7 100644 --- a/test/s3tables/catalog/duckdb_deterministic_location_test.go +++ b/test/s3tables/catalog/duckdb_deterministic_location_test.go @@ -115,6 +115,9 @@ SELECT 'duckdb deterministic-location describe ok' AS marker; t.Logf("DuckDB output:\n%s", outputStr) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } // Tolerate environments where the Iceberg extension/version is too // limited to exercise this code path. The intent is to gain // coverage where it's available, not to require it everywhere. diff --git a/test/s3tables/catalog/duckdb_oauth_test.go b/test/s3tables/catalog/duckdb_oauth_test.go index 346cd2d096..aa151f48e3 100644 --- a/test/s3tables/catalog/duckdb_oauth_test.go +++ b/test/s3tables/catalog/duckdb_oauth_test.go @@ -351,6 +351,9 @@ SELECT 'DuckDB Iceberg OAuth secret created successfully' as result; t.Logf("DuckDB output:\n%s", outputStr) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } if strings.Contains(outputStr, "iceberg extension is not available") || strings.Contains(outputStr, "Failed to load") { t.Skip("Skipping: Iceberg extension not available in DuckDB Docker image") @@ -379,6 +382,9 @@ SELECT 'DuckDB Iceberg OAuth secret created successfully' as result; t.Logf("DuckDB fallback output:\n%s", fallbackStr) if fallbackErr != nil { + if testutil.IsIllegalInstruction(fallbackErr) { + t.Skipf("DuckDB image cannot run on this CPU: %v", fallbackErr) + } if strings.Contains(fallbackStr, "NotFound_404") && strings.Contains(fallbackStr, "/v1/oauth/tokens") { t.Fatal("OAuth token endpoint returned 404 - the fix is not working") } diff --git a/test/s3tables/catalog/iceberg_catalog_test.go b/test/s3tables/catalog/iceberg_catalog_test.go index 908f54b176..d8490fa629 100644 --- a/test/s3tables/catalog/iceberg_catalog_test.go +++ b/test/s3tables/catalog/iceberg_catalog_test.go @@ -574,6 +574,9 @@ SELECT 'Iceberg extension loaded successfully' as result; output, err := cmd.CombinedOutput() if err != nil { t.Logf("DuckDB output: %s", output) + if testutil.IsIllegalInstruction(err) { + t.Skipf("DuckDB image cannot run on this CPU: %v", err) + } // Check for expected errors in certain CI environments outputStr := string(output) if strings.Contains(outputStr, "iceberg extension is not available") || diff --git a/test/s3tables/catalog/issue_9103_test.go b/test/s3tables/catalog/issue_9103_test.go index be3dddbd13..0d2f86b360 100644 --- a/test/s3tables/catalog/issue_9103_test.go +++ b/test/s3tables/catalog/issue_9103_test.go @@ -218,6 +218,9 @@ SELECT * FROM test_catalog.%s.%s LIMIT 0; } if runErr != nil { + if testutil.IsIllegalInstruction(runErr) { + t.Skipf("DuckDB image cannot run on this CPU: %v", runErr) + } if strings.Contains(outStr, "does not exist because schema") && strings.Contains(outStr, namespace) { t.Fatalf("reproduced issue #9103: DuckDB cannot see namespace %q after ATTACH 's3://%s/'; see output above", diff --git a/test/s3tables/catalog/lance_client_test.go b/test/s3tables/catalog/lance_client_test.go index e44d032060..a9abac6f42 100644 --- a/test/s3tables/catalog/lance_client_test.go +++ b/test/s3tables/catalog/lance_client_test.go @@ -2,11 +2,13 @@ package catalog import ( "fmt" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" "os" "os/exec" "path/filepath" "testing" + + "github.com/seaweedfs/seaweedfs/test/testutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3tables" ) // TestLanceNamespaceRealClient drives the namespace with the Lance client rather @@ -58,6 +60,9 @@ func TestLanceNamespaceRealClient(t *testing.T) { t.Logf("Running Lance client test against %s", namespaceURL) if err := cmd.Run(); err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("Lance client image cannot run on this CPU: %v", err) + } t.Errorf("Lance client test failed: %v", err) } } diff --git a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go index 96613937c0..be45fd7f18 100644 --- a/test/s3tables/catalog_lancedb/lancedb_catalog_test.go +++ b/test/s3tables/catalog_lancedb/lancedb_catalog_test.go @@ -272,6 +272,9 @@ func (env *environment) runClient(t *testing.T, bucket string) { out, err := cmd.CombinedOutput() t.Logf("LanceDB client output:\n%s", out) if err != nil { + if testutil.IsIllegalInstruction(err) { + t.Skipf("LanceDB client image cannot run on this CPU: %v", err) + } t.Fatalf("the LanceDB client failed: %v", err) } if !strings.Contains(string(out), "PASS") { diff --git a/test/s3tables/catalog_risingwave/setup_test.go b/test/s3tables/catalog_risingwave/setup_test.go index 155c390eed..f2adf83433 100644 --- a/test/s3tables/catalog_risingwave/setup_test.go +++ b/test/s3tables/catalog_risingwave/setup_test.go @@ -194,6 +194,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) { if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to start RisingWave container: %v\n%s", err, string(output)) } + env.checkRisingWaveContainer(t) // Start a sidecar postgres container for running psql commands sidecarName := "seaweed-risingwave-sidecar-" + randomString(8) @@ -205,6 +206,7 @@ func (env *TestEnvironment) StartRisingWave(t *testing.T) { "sleep", "infinity", ) if output, err := sidecarCmd.CombinedOutput(); err != nil { + env.checkRisingWaveContainer(t) t.Fatalf("failed to start postgres sidecar: %v\n%s", err, string(output)) } @@ -220,6 +222,7 @@ func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool { deadline := time.Now().Add(timeout) env.t.Logf(">>> Waiting for RisingWave to be ready (timeout %v)...\n", timeout) for time.Now().Before(deadline) { + env.checkRisingWaveContainer(env.t) if output, err := runPostgresClientSQL(env.postgresSidecar, "SELECT 1;"); err == nil { env.t.Logf(">>> RisingWave is ready.\n") return true @@ -231,6 +234,26 @@ func (env *TestEnvironment) waitForRisingWave(timeout time.Duration) bool { return false } +func (env *TestEnvironment) checkRisingWaveContainer(t *testing.T) { + t.Helper() + out, err := exec.Command("docker", "inspect", "--format", "{{.State.Running}} {{.State.ExitCode}} {{.State.OOMKilled}}", env.risingwaveContainer).CombinedOutput() + if err != nil { + t.Fatalf("inspect RisingWave container: %v\n%s", err, out) + } + state := strings.Fields(string(out)) + if len(state) != 3 { + t.Fatalf("unexpected RisingWave container state: %s", out) + } + if state[0] == "true" { + return + } + if state[1] == "132" && state[2] == "false" { + t.Skip("RisingWave image exited with illegal instruction on this CPU") + } + logs, _ := exec.Command("docker", "logs", env.risingwaveContainer).CombinedOutput() + t.Fatalf("RisingWave container exited before it became ready (code %s, OOM killed: %s):\n%s", state[1], state[2], logs) +} + func runPostgresClientSQL(containerName, sql string) ([]byte, error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/test/testutil/helpers.go b/test/testutil/helpers.go index 8d2d6b2806..e0eafb33f3 100644 --- a/test/testutil/helpers.go +++ b/test/testutil/helpers.go @@ -51,6 +51,11 @@ func RequireImageRunnable(t *testing.T, args ...string) { } } +func IsIllegalInstruction(err error) bool { + var exitErr *exec.ExitError + return errors.As(err, &exitErr) && exitErr.ExitCode() == 132 +} + func FindBindIP() string { addrs, err := net.InterfaceAddrs() if err != nil { diff --git a/test/testutil/helpers_test.go b/test/testutil/helpers_test.go new file mode 100644 index 0000000000..efc302fc6c --- /dev/null +++ b/test/testutil/helpers_test.go @@ -0,0 +1,39 @@ +package testutil + +import ( + "errors" + "fmt" + "os" + "os/exec" + "strconv" + "testing" +) + +func TestIsIllegalInstruction(t *testing.T) { + if code := os.Getenv("SEAWEEDFS_TEST_EXIT_CODE"); code != "" { + status, _ := strconv.Atoi(code) + os.Exit(status) + } + + for _, tc := range []struct { + name string + code string + want bool + }{ + {"illegal instruction", "132", true}, + {"ordinary failure", "1", false}, + {"out of memory", "137", false}, + } { + t.Run(tc.name, func(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestIsIllegalInstruction$") + cmd.Env = append(os.Environ(), "SEAWEEDFS_TEST_EXIT_CODE="+tc.code) + err := cmd.Run() + if got := IsIllegalInstruction(fmt.Errorf("docker run: %w", err)); got != tc.want { + t.Fatalf("IsIllegalInstruction(%v) = %v, want %v", err, got, tc.want) + } + }) + } + if IsIllegalInstruction(nil) || IsIllegalInstruction(errors.New("exit status 132")) { + t.Fatal("non-process errors must not be treated as illegal instructions") + } +} From 5e01caa52286c8baf0aeff8a205c953a9e80632a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 22:41:36 -0700 Subject: [PATCH 42/70] master: cap parallel vacuums per rack via master.vacuum.max_parallel_per_rack (#11935) * master: cap parallel vacuums per rack via -maxParallelVacuumPerRack Hosts that run one volume server per disk under a single rack label could not bound host-level vacuum I/O: maxParallelVacuumPerServer limits each process, so every disk on the machine compacted at once. A second quota map keyed on rack id debits alongside the per-server one, so at most N volume copies compact in parallel within a rack. Defaults to 0 (no limit); the flag is wired on master, server, mini and master.follower. * master: resolve a data node's rack key once per vacuum sweep A volume server that unregisters mid-sweep loses its parent node, so the quota credit path calling dn.GetRack() panicked and crashed the master. Rack names are also local to their data center, so same-named racks in different centers shared one quota bucket. Each node's rack key is now captured when the limiters are built and carries the data center id, so checks, debits and credits never touch the live parent pointer. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: reserve vacuum quota per needed copy count, not per copy Each copy of a volume checked the shared server/rack quota independently, so several same-rack replicas all passed against one remaining slot and the debits drove the bucket negative. Needs are now counted per server and per rack first; a volume whose own copies exceed a cap reserves the whole bucket once it is free instead of pending forever. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master.follower: register -maxParallelVacuumPerRack as a flag It was only a fixed default, so passing the flag on the command line failed flag parsing. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: key the rack vacuum quota by rack node identity Joining data center and rack names with "/" still let two different racks share one bucket when the names themselves contained slashes. Key the limiter by the rack node instead, which is unique per rack regardless of naming. * master: keep the tiered-replica vacuum test through the rebase * master: document 0 as no limit for -maxParallelVacuumPerServer The clamped per-server need makes a non-positive cap mean "no limit", matching the new rack flag; say so in the flag help. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: key the rack-key cache by data node identity Node ids are only unique within a rack, so two servers that share an id in different racks would have reused the first server's cached rack and escaped the per-rack vacuum cap. * master: read the per-rack vacuum cap from master.toml -maxParallelVacuumPerRack becomes the master.max_parallel_vacuum_per_rack config key, so the new master limit is set the same way as other master options instead of a flag wired into every command that embeds a master. * master: let master.max_parallel_vacuum_per_server override the flag The released -maxParallelVacuumPerServer flag keeps working, but a value in master.toml now wins when set, matching how master.mdir already overrides -mdir. * master: group the vacuum caps under master.vacuum in master.toml Rename master.max_parallel_vacuum_per_rack to master.vacuum.max_parallel_per_rack and master.max_parallel_vacuum_per_server to master.vacuum.max_parallel_per_server, so the keys sit in one subsystem table like master.volume_growth and master.maintenance. * master: deprecate -maxParallelVacuumPerServer for master.toml The flag still works - and is overridden by master.vacuum.max_parallel_per_server when set - but the help text now points operators at the config key, matching the codebase style of help-text-only deprecation. * topology: read node parent atomically UnlinkChildNode clears a data node's parent under the rack's lock while readers like the vacuum sweep's rack capture reach it with no shared lock, so a torn interface read could crash the master. Store the parent in an atomic.Pointer and nil-guard the detached walk in DataNode.GetTopology. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/master.go | 2 +- weed/command/mini.go | 2 +- weed/command/scaffold/master.toml | 9 + weed/command/server.go | 2 +- weed/server/master_grpc_server_volume.go | 2 +- weed/server/master_server.go | 7 + weed/server/master_server_handlers_admin.go | 2 +- weed/topology/data_node.go | 4 +- weed/topology/node.go | 32 ++- weed/topology/topology_event_handling.go | 4 +- weed/topology/topology_vacuum.go | 81 +++++-- weed/topology/topology_vacuum_test.go | 246 +++++++++++++++++++- 12 files changed, 347 insertions(+), 46 deletions(-) diff --git a/weed/command/master.go b/weed/command/master.go index 2a5a409678..b12f0a6210 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -95,7 +95,7 @@ func init() { m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)") m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.") - m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server") + m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") m.maxWritableVolumes = cmdMaster.Flag.Int("maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") diff --git a/weed/command/mini.go b/weed/command/mini.go index 381e438e9c..0d2660438e 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -445,7 +445,7 @@ func initMiniMasterFlags() { miniMasterOptions.peers = cmdMini.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list (default: none for single master)") miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)") miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") - miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") miniMasterOptions.maxWritableVolumes = cmdMini.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index b7a344e231..3e06fa2259 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -11,6 +11,15 @@ scripts = """{{DEFAULT_MAINTENANCE_SCRIPTS}}""" sleep_minutes = 17 # sleep minutes between each script execution +# vacuum reclaims deleted space by compacting volumes +[master.vacuum] +# concurrent volume compactions per volume server; overrides the deprecated +# -maxParallelVacuumPerServer flag (default 1) when set; 0 = no limit +# max_parallel_per_server = 1 +# concurrent volume copies compacting within one rack; 0 = no limit +max_parallel_per_rack = 0 + + [master.sequencer] type = "raft" # Choose [raft|snowflake] type for storing the file id sequence # when sequencer.type = snowflake, the snowflake id must be different from other masters diff --git a/weed/command/server.go b/weed/command/server.go index 0277ccf421..5e41c6e3bb 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -97,7 +97,7 @@ func init() { masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list") masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") - masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server") + masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") masterOptions.maxWritableVolumes = cmdServer.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") diff --git a/weed/server/master_grpc_server_volume.go b/weed/server/master_grpc_server_volume.go index 53bd2c27f0..de1e54333c 100644 --- a/weed/server/master_grpc_server_volume.go +++ b/weed/server/master_grpc_server_volume.go @@ -399,7 +399,7 @@ func (ms *MasterServer) VacuumVolume(ctx context.Context, req *master_pb.VacuumV resp := &master_pb.VacuumVolumeResponse{} - ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.option.MaxParallelVacuumPerServer, req.VolumeId, req.Collection, ms.preallocateSize, false, 0) + ms.Topo.Vacuum(ms.grpcDialOption, float64(req.GarbageThreshold), ms.option.MaxParallelVacuumPerServer, ms.option.MaxParallelVacuumPerRack, req.VolumeId, req.Collection, ms.preallocateSize, false, 0) return resp, nil } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 6a5c5e7ac9..dec88ab154 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -54,6 +54,7 @@ type MasterOption struct { FileSizeLimitMB int VolumePreallocate bool MaxParallelVacuumPerServer int + MaxParallelVacuumPerRack int MaxWritableVolumes int VacuumIntervalSeconds int VacuumDeleteEmptyAfterSeconds int @@ -128,6 +129,11 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se option.VolumeGrowthDisabled = v.GetBool("master.volume_growth.disable") v.SetDefault("master.volume_growth.freeze_on_topology_mismatch", true) v.SetDefault("master.volume_growth.state_loss_recovery_wait", "0s") + v.SetDefault("master.vacuum.max_parallel_per_rack", 0) + option.MaxParallelVacuumPerRack = v.GetInt("master.vacuum.max_parallel_per_rack") + if v.IsSet("master.vacuum.max_parallel_per_server") { + option.MaxParallelVacuumPerServer = v.GetInt("master.vacuum.max_parallel_per_server") + } topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1") topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2") @@ -230,6 +236,7 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se ms.grpcDialOption, ms.option.GarbageThreshold, ms.option.MaxParallelVacuumPerServer, + ms.option.MaxParallelVacuumPerRack, topology.VolumeGrowStrategy.Threshold, ms.preallocateSize, time.Duration(ms.option.VacuumIntervalSeconds)*time.Second, diff --git a/weed/server/master_server_handlers_admin.go b/weed/server/master_server_handlers_admin.go index 293043d679..e6479ea49a 100644 --- a/weed/server/master_server_handlers_admin.go +++ b/weed/server/master_server_handlers_admin.go @@ -61,7 +61,7 @@ func (ms *MasterServer) volumeVacuumHandler(w http.ResponseWriter, r *http.Reque } } // glog.Infoln("garbageThreshold =", gcThreshold) - ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.option.MaxParallelVacuumPerServer, 0, "", ms.preallocateSize, false, 0) + ms.Topo.Vacuum(ms.grpcDialOption, gcThreshold, ms.option.MaxParallelVacuumPerServer, ms.option.MaxParallelVacuumPerRack, 0, "", ms.preallocateSize, false, 0) ms.dirStatusHandler(w, r) } diff --git a/weed/topology/data_node.go b/weed/topology/data_node.go index 731e607091..444adf7646 100644 --- a/weed/topology/data_node.go +++ b/weed/topology/data_node.go @@ -358,10 +358,10 @@ func (dn *DataNode) matchesLocation(option *VolumeGrowOption) bool { func (dn *DataNode) GetTopology() *Topology { p := dn.Parent() - for p.Parent() != nil { + for p != nil && p.Parent() != nil { p = p.Parent() } - t := p.(*Topology) + t, _ := p.(*Topology) return t } diff --git a/weed/topology/node.go b/weed/topology/node.go index 98e212ba07..be642027e7 100644 --- a/weed/topology/node.go +++ b/weed/topology/node.go @@ -145,9 +145,12 @@ type Node interface { } type NodeImpl struct { - diskUsages *DiskUsages - id NodeId - parent Node + diskUsages *DiskUsages + id NodeId + // parent is an atomic pointer because unlinking runs under the rack's + // lock while readers (vacuum sweeps, usage propagation, String) reach + // it with no shared lock. + parent atomic.Pointer[Node] sync.RWMutex // lock children children map[NodeId]Node // maxVolumeId uses atomic ops so UpAdjustMaxVolumeId (called from the @@ -292,8 +295,8 @@ func (n *NodeImpl) IsLocked() (isTryLock bool) { } func (n *NodeImpl) String() string { - if n.parent != nil { - return n.parent.String() + ":" + string(n.id) + if parent := n.Parent(); parent != nil { + return parent.String() + ":" + string(n.id) } return string(n.id) } @@ -390,7 +393,11 @@ func (n *NodeImpl) ReleaseReservedCapacity(reservationId string) { } } func (n *NodeImpl) SetParent(node Node) { - n.parent = node + if node == nil { + n.parent.Store(nil) + return + } + n.parent.Store(&node) } func (n *NodeImpl) Children() (ret []Node) { @@ -403,7 +410,10 @@ func (n *NodeImpl) Children() (ret []Node) { } func (n *NodeImpl) Parent() Node { - return n.parent + if parent := n.parent.Load(); parent != nil { + return *parent + } + return nil } func (n *NodeImpl) GetValue() interface{} { @@ -482,8 +492,8 @@ func (n *NodeImpl) reserveOneVolumeInternal(r int64, option *VolumeGrowOption, u func (n *NodeImpl) UpAdjustDiskUsageDelta(diskType types.DiskType, diskUsage *DiskUsageCounts) { //can be negative existingDisk := n.getOrCreateDisk(diskType) existingDisk.addDiskUsageCounts(diskUsage) - if n.parent != nil { - n.parent.UpAdjustDiskUsageDelta(diskType, diskUsage) + if parent := n.Parent(); parent != nil { + parent.UpAdjustDiskUsageDelta(diskType, diskUsage) } } func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) { @@ -497,8 +507,8 @@ func (n *NodeImpl) UpAdjustMaxVolumeId(vid needle.VolumeId) { break } } - if n.parent != nil { - n.parent.UpAdjustMaxVolumeId(vid) + if parent := n.Parent(); parent != nil { + parent.UpAdjustMaxVolumeId(vid) } } func (n *NodeImpl) GetMaxVolumeId() needle.VolumeId { diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go index a0df67e221..9dcd6f93e3 100644 --- a/weed/topology/topology_event_handling.go +++ b/weed/topology/topology_event_handling.go @@ -13,7 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/storage" ) -func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, growThreshold float64, preallocate int64, vacuumInterval time.Duration, deleteEmptyAfter time.Duration) { +func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, garbageThreshold float64, concurrentVacuumLimitPerVolumeServer int, concurrentVacuumLimitPerRack int, growThreshold float64, preallocate int64, vacuumInterval time.Duration, deleteEmptyAfter time.Duration) { if vacuumInterval <= 0 { vacuumInterval = 14 * time.Minute } @@ -39,7 +39,7 @@ func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, g t.EnableVacuumByPlugin() } if !t.IsVacuumDisabled() { - t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, 0, "", preallocate, true, deleteEmptyAfter) + t.Vacuum(grpcDialOption, garbageThreshold, concurrentVacuumLimitPerVolumeServer, concurrentVacuumLimitPerRack, 0, "", preallocate, true, deleteEmptyAfter) } } else { stats.MasterReplicaPlacementMismatch.Reset() diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 83e1ab6f47..0cc779d0ea 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -251,7 +251,7 @@ func (t *Topology) batchVacuumVolumeCleanup(grpcDialOption grpc.DialOption, vl * } } -func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, maxParallelVacuumPerServer int, volumeId uint32, collection string, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { +func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float64, maxParallelVacuumPerServer int, maxParallelVacuumPerRack int, volumeId uint32, collection string, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { // if there is vacuum going on, return immediately swapped := atomic.CompareAndSwapInt64(&t.vacuumLockCounter, 0, 1) @@ -285,7 +285,7 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, false) } } else { - t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, maxParallelVacuumPerServer, preallocate, automatic, deleteEmptyAfter) + t.vacuumOneVolumeLayout(grpcDialOption, volumeLayout, c, garbageThreshold, maxParallelVacuumPerServer, maxParallelVacuumPerRack, preallocate, automatic, deleteEmptyAfter) } } if automatic && t.IsVacuumDisabled() { @@ -299,7 +299,7 @@ func (t *Topology) Vacuum(grpcDialOption grpc.DialOption, garbageThreshold float } } -func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, maxParallelVacuumPerServer int, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { +func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeLayout *VolumeLayout, c *Collection, garbageThreshold float64, maxParallelVacuumPerServer int, maxParallelVacuumPerRack int, preallocate int64, automatic bool, deleteEmptyAfter time.Duration) { volumeLayout.accessLock.RLock() todoVolumeMap := make(map[needle.VolumeId]*VolumeLocationList) @@ -314,14 +314,27 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL t.deleteEmptyVolumes(grpcDialOption, volumeLayout, todoVolumeMap, deleteEmptyAfter) } - // limiter for each volume server + // limiter for each volume server, and for each rack when a rack cap is + // set — several volume servers can share one rack's disks and network. + // Rack keys are resolved once up front: a node that unregisters mid-sweep + // loses its parent, and a later GetRack call would panic. limiter := make(map[NodeId]int) + rackLimiter := make(map[Node]int) + dnRack := make(map[*DataNode]Node) var limiterLock sync.Mutex for _, locationList := range todoVolumeMap { for _, dn := range locationList.list { if _, ok := limiter[dn.Id()]; !ok { limiter[dn.Id()] = maxParallelVacuumPerServer } + if maxParallelVacuumPerRack > 0 { + if _, ok := dnRack[dn]; !ok { + dnRack[dn] = vacuumRackKey(dn) + } + if _, ok := rackLimiter[dnRack[dn]]; !ok { + rackLimiter[dnRack[dn]] = maxParallelVacuumPerRack + } + } } } @@ -333,38 +346,59 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL for len(todoVolumeMap) > 0 { pendingVolumeMap := make(map[needle.VolumeId]*VolumeLocationList) for vid, locationList := range todoVolumeMap { - hasEnoughQuota := true + // copies on the same server or rack reserve together; a volume + // that alone exceeds a cap waits for the whole bucket to be free + // instead of pending forever + serverNeeds := make(map[NodeId]int) + rackNeeds := make(map[Node]int) for _, dn := range locationList.list { - limiterLock.Lock() - quota := limiter[dn.Id()] - limiterLock.Unlock() - if quota <= 0 { + serverNeeds[dn.Id()]++ + if maxParallelVacuumPerRack > 0 { + rackNeeds[dnRack[dn]]++ + } + } + limiterLock.Lock() + hasEnoughQuota := true + for id, need := range serverNeeds { + serverNeeds[id] = min(need, maxParallelVacuumPerServer) + if limiter[id] < serverNeeds[id] { + hasEnoughQuota = false + } + } + for id, need := range rackNeeds { + rackNeeds[id] = min(need, maxParallelVacuumPerRack) + if rackLimiter[id] < rackNeeds[id] { hasEnoughQuota = false - break } } if !hasEnoughQuota { + limiterLock.Unlock() pendingVolumeMap[vid] = locationList continue } // debit the quota - for _, dn := range locationList.list { - limiterLock.Lock() - limiter[dn.Id()]-- - limiterLock.Unlock() + for id, need := range serverNeeds { + limiter[id] -= need } + for id, need := range rackNeeds { + rackLimiter[id] -= need + } + limiterLock.Unlock() wg.Add(1) executor.Execute(func() { defer wg.Done() t.vacuumOneVolumeId(grpcDialOption, volumeLayout, c, garbageThreshold, locationList, vid, preallocate, true) // credit the quota - for _, dn := range locationList.list { - limiterLock.Lock() - limiter[dn.Id()]++ - limiterLock.Unlock() + limiterLock.Lock() + for id, need := range serverNeeds { + limiter[id] += need } + for id, need := range rackNeeds { + rackLimiter[id] += need + } + limiterLock.Unlock() select { case quotaFreed <- struct{}{}: default: @@ -390,6 +424,17 @@ func (t *Topology) vacuumOneVolumeLayout(grpcDialOption grpc.DialOption, volumeL } +// vacuumRackKey identifies a data node's rack by the rack node itself, so +// racks that share a name in different data centers still get separate +// quotas. A node that already lost its parent falls back to itself, +// degrading to the per-server cap. +func vacuumRackKey(dn *DataNode) Node { + if rack := dn.Parent(); rack != nil { + return rack + } + return dn +} + // skipReadOnly is set by the background scan and all-volumes sweep, where a // read-only flag usually means an unhealthy disk. Even then a copy that is // read-only because its disk is low on space stays eligible — compaction is diff --git a/weed/topology/topology_vacuum_test.go b/weed/topology/topology_vacuum_test.go index 82643f33b2..106011d594 100644 --- a/weed/topology/topology_vacuum_test.go +++ b/weed/topology/topology_vacuum_test.go @@ -257,13 +257,27 @@ func TestDeleteEmptyVolumesKeepsVidWhenCopyDeleteFails(t *testing.T) { type fakeVacuumServer struct { volume_server_pb.UnimplementedVolumeServerServer - mu sync.Mutex - checks map[uint32]*volume_server_pb.VacuumVolumeCheckResponse - committed []uint32 - compactErr error - hang string - entered chan string - cancelled chan struct{} + mu sync.Mutex + checks map[uint32]*volume_server_pb.VacuumVolumeCheckResponse + committed []uint32 + compactErr error + hang string + entered chan string + cancelled chan struct{} + compactGate chan struct{} + compactTracker *compactTracker +} + +type compactTracker struct { + mu sync.Mutex + inFlight int + max int +} + +func (t *compactTracker) maxInFlight() int { + t.mu.Lock() + defer t.mu.Unlock() + return t.max } func (f *fakeVacuumServer) stall(ctx context.Context, phase string) error { @@ -299,6 +313,28 @@ func (f *fakeVacuumServer) VacuumVolumeCompact(_ *volume_server_pb.VacuumVolumeC if f.hang == "compact" { return f.stall(stream.Context(), "compact") } + if f.compactGate != nil { + select { + case f.entered <- "compact": + default: + } + f.compactTracker.mu.Lock() + f.compactTracker.inFlight++ + if f.compactTracker.inFlight > f.compactTracker.max { + f.compactTracker.max = f.compactTracker.inFlight + } + f.compactTracker.mu.Unlock() + defer func() { + f.compactTracker.mu.Lock() + f.compactTracker.inFlight-- + f.compactTracker.mu.Unlock() + }() + select { + case <-f.compactGate: + case <-stream.Context().Done(): + return stream.Context().Err() + } + } return f.compactErr } @@ -456,7 +492,7 @@ func TestVacuumStalledVolumeServerReleasesGuard(t *testing.T) { done := make(chan struct{}) go func() { - topo.Vacuum(dialOption, 0.3, 1, 1, "c", 0, false, 0) + topo.Vacuum(dialOption, 0.3, 1, 0, 1, "c", 0, false, 0) close(done) }() @@ -530,3 +566,197 @@ func TestVacuumSkipsTieredReplica(t *testing.T) { t.Fatalf("committed %v, want only the local volume 2", fake.committed) // volume 2 is the positive control } } + +// A rack cap bounds copies compacting in parallel inside one rack even when +// every data node still has per-server quota left: volumes whose only copy +// sits on different data nodes of the same rack serialize. +func TestVacuumRackLimitSerializesVolumes(t *testing.T) { + tracker := &compactTracker{} + newFake := func(vid uint32) *fakeVacuumServer { + return &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{vid: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: tracker, + } + } + fake1, fake2 := newFake(1), newFake(2) + port1, dialOption := startFakeVolumeServer(t, fake1) + port2, _ := startFakeVolumeServer(t, fake2) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1") + dns := []*DataNode{ + rack.GetOrCreateDataNode("127.0.0.1", 8080, port1, "127.0.0.1", "dn-1", map[string]uint32{"": 10}), + rack.GetOrCreateDataNode("127.0.0.1", 8081, port2, "127.0.0.1", "dn-2", map[string]uint32{"": 10}), + } + for i, dn := range dns { + v := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + } + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 10, 1, 0, "c", 0, false, 0) + close(done) + }() + + deadline := time.After(15 * time.Second) + var winner *fakeVacuumServer + for winner == nil { + select { + case <-fake1.entered: + winner = fake1 + case <-fake2.entered: + winner = fake2 + case <-deadline: + t.Fatal("no volume reached the compact phase") + } + } + + // while the first compact is gated, the rack cap must keep the second + // volume from starting one + time.Sleep(300 * time.Millisecond) + if got := tracker.maxInFlight(); got != 1 { + t.Fatalf("in-flight compacts = %d, want the rack cap to hold it at 1", got) + } + + close(fake1.compactGate) + close(fake2.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the gates opened") + } + for _, fake := range []*fakeVacuumServer{fake1, fake2} { + fake.mu.Lock() + n := len(fake.committed) + fake.mu.Unlock() + if n != 1 { + t.Fatalf("committed volumes = %d, want 1", n) + } + } +} + +// Racks that share a name in different data centers are independent: a per +// rack cap must not serialize volumes sitting in dc1/rack1 and dc2/rack1. +func TestVacuumRackLimitScopesRacksByDataCenter(t *testing.T) { + tracker := &compactTracker{} + newFake := func(vid uint32) *fakeVacuumServer { + return &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{vid: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: tracker, + } + } + fake1, fake2 := newFake(1), newFake(2) + port1, dialOption := startFakeVolumeServer(t, fake1) + port2, _ := startFakeVolumeServer(t, fake2) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + dns := []*DataNode{ + topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, port1, "127.0.0.1", "dn-1", map[string]uint32{"": 10}), + topo.GetOrCreateDataCenter("dc2").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8081, port2, "127.0.0.1", "dn-2", map[string]uint32{"": 10}), + } + for i, dn := range dns { + v := storage.VolumeInfo{ + Id: needle.VolumeId(i + 1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + } + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 10, 1, 0, "c", 0, false, 0) + close(done) + }() + + // both volumes sit in different racks, so both compacts start while the + // gates stay closed + deadline := time.After(15 * time.Second) + for _, fake := range []*fakeVacuumServer{fake1, fake2} { + select { + case <-fake.entered: + case <-deadline: + t.Fatal("a volume in an independent rack never reached the compact phase") + } + } + + close(fake1.compactGate) + close(fake2.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the gates opened") + } +} + +// A data node that unregisters while its volume is still compacting must not +// crash the sweep when the quota is credited back: the rack key was captured +// before the RPCs ran. +func TestVacuumRackLimitSurvivesUnregister(t *testing.T) { + fake := &fakeVacuumServer{ + checks: map[uint32]*volume_server_pb.VacuumVolumeCheckResponse{1: {GarbageRatio: 0.9}}, + entered: make(chan string, 1), + compactGate: make(chan struct{}), + compactTracker: &compactTracker{}, + } + grpcPort, dialOption := startFakeVolumeServer(t, fake) + + topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false) + dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1"). + GetOrCreateDataNode("127.0.0.1", 8080, grpcPort, "127.0.0.1", "dn-1", map[string]uint32{"": 10}) + v := storage.VolumeInfo{ + Id: needle.VolumeId(1), + Size: 1 << 20, + Collection: "c", + ModifiedAtSecond: time.Now().Unix(), + Version: needle.GetCurrentVersion(), + ReplicaPlacement: &super_block.ReplicaPlacement{}, + Ttl: needle.EMPTY_TTL, + } + dn.UpdateVolumes([]storage.VolumeInfo{v}) + topo.RegisterVolumeLayout(v, dn) + + done := make(chan struct{}) + go func() { + topo.Vacuum(dialOption, 0.3, 1, 1, 0, "c", 0, false, 0) + close(done) + }() + + select { + case <-fake.entered: + case <-time.After(15 * time.Second): + t.Fatal("volume never reached the compact phase") + } + + topo.UnRegisterDataNode(dn) + close(fake.compactGate) + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("vacuum did not finish after the data node unregistered") + } +} From 57a122fbcfbbc2e834db78b04186cbc9d7b0db34 Mon Sep 17 00:00:00 2001 From: jsas <1351492+jsas@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:43:39 -0700 Subject: [PATCH 43/70] filer.remote.sync: dispatch an update event by the side of the mount it touches instead of slicing the mount prefix off a path outside it (#11958) * filer.remote.sync: dispatch an update event by the side of the mount it touches instead of slicing the mount prefix off a path outside it The daemon subscribes to its mount prefix and to /etc/remote. An update event whose path lies outside the mount still reaches processUpdateEvent: an update of the /etc/remote directory entry itself (which fs.meta.load of a mapping dump re-creates) through the /etc/remote prefix, or a rename across the mount boundary through the path on the inside. processUpdateEvent then maps both paths to remote keys by slicing the mount prefix off, which panics with "slice bounds out of range [20:11]" on a path shorter than the mount and names a key in the wrong tree on a longer one. The event sits in the replay window, so every restart dies on it and the persisted offset never moves. Before dispatch, an update event is now mapped onto the side the mount covers: a rename into the mount is the creation of the new entry, a rename out of it the deletion of the old entry's object (kept when the entry is remote-only and the object holds its only copy), an event with neither path inside is not the mount's and is dropped, and an update with both paths inside is unchanged. * filer.remote.sync: extract processCreateEvent Moves the create branch of eachEntryFunc into a function, matching processDeleteEvent and processUpdateEvent, so the create path can be exercised directly in tests. No behavior change. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.remote.sync: upload a create reshaped from a rename into the mount A file synced under the mount keeps its RemoteEntry across renames. The rename out deletes the remote object, and the rename back in reshapes to a create whose inherited stamp still matches the entry, so shouldSendToRemote skipped the upload and the new key stayed empty. The reshaped create now sends unconditionally: the stamp describes the old key, not this one. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer.remote.sync: a create reshaped from a rename into the mount writes an empty entry as a zero-byte object * filer.remote.sync: a create reshaped from a rename sends a stamped entry truncated since its sync A stamped entry still reports remote-only once truncated to zero (the stamp's RemoteSize stays positive), so the remote-only exemption skipped it and the new key got no zero-byte object. Gate the exemption on isDroppedLocalCopy: only an entry whose size still matches its stamp has its content already at the new key via the rename's remote copy. * filer.remote.sync: recover a superseded create reshaped from a rename-in A create reshaped from a rename into the mount whose write is superseded was skipped like a plain create, but no later event covers this destination: remote.uncache accepts the rename's inherited RemoteEntry stamp and drops the local chunks, and the update it logs skips the upload through isDroppedLocalCopy, while the source mount's sync still deletes the source object. The destination object is then missing and the content exists nowhere. recoverSupersededCreate finishes the event from the entry the filer holds now: a rewritten entry is left to its own event's upload or uploaded here on the stamp edge, and a remote-only entry is completed by copying the object its stamp describes -- resolved through a pending-rename marker under this mount, or through the filer's mount mappings to the source mount's remote location. --------- Co-authored-by: james Co-authored-by: Chris Lu Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf --- weed/command/filer_remote_sync_dir.go | 373 +++++++++++++++++---- weed/command/filer_remote_sync_dir_test.go | 358 +++++++++++++++++++- 2 files changed, 672 insertions(+), 59 deletions(-) diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index 90d7514222..38261cac68 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -140,63 +140,24 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem if filer_pb.IsEmpty(resp) { return nil } + mustSend := false + var renameSrc util.FullPath + if message.OldEntry != nil && message.NewEntry != nil { + reshaped, inMount := reshapeCrossMountRename(mountedDir, resp) + if !inMount { + return nil + } + // a create reshaped from a rename into the mount can carry a + // RemoteEntry earned at the old path: it stamps the old key, so + // it cannot vouch for this destination + mustSend = filer_pb.IsCreate(reshaped) + if mustSend { + renameSrc = util.NewFullPath(resp.Directory, message.OldEntry.Name) + } + resp, message = reshaped, reshaped.EventNotification + } if filer_pb.IsCreate(resp) { - if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { - return nil - } - if isSystemPath(message.NewParentPath, message.NewEntry.Name) { - glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) - return nil - } - // Propagate delete markers as deletions on the remote. - // Delete markers are zero-content version entries, so they - // would be filtered out by the HasData check below. - if isDeleteMarker(message.NewEntry) { - if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok { - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation) - return syncDeleteMarker(client, option, message, dest) - } - return nil - } - if !filer.HasData(message.NewEntry) { - return nil - } - glog.V(2).Infof("create: %+v", resp) - if !shouldSendToRemote(message.NewEntry) { - glog.V(2).Infof("skipping creating: %+v", resp) - return nil - } - // Rewrite internal versioning paths to the original S3 key - // to prevent double-versioning when central also has versioning enabled - parentPath, entryName := message.NewParentPath, message.NewEntry.Name - isRewrittenVersion := false - if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok { - glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName) - parentPath, entryName = newParent, newName - isRewrittenVersion = true - } - dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation) - if message.NewEntry.IsDirectory { - glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) - return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, *option.storageClass)) - } - remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, *option.storageClass), dest) - if errors.Is(writeErr, errSuperseded) { - glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) - return nil - } - if writeErr != nil { - return writeErr - } - // Skip updateLocalEntry for versioned rewrites: the logical - // object (e.g. file.xml) has no filer entry in versioned - // buckets, and stamping the internal v_* entry with a - // RemoteEntry for the logical key is semantically wrong. - // Replay is safe because S3 PutObject is idempotent. - if isRewrittenVersion { - return nil - } - return updateLocalEntry(option, message.NewParentPath, message.NewEntry, remoteEntry) + return processCreateEvent(option, filerSource, *option.storageClass, client, mountedDir, remoteStorageMountLocation, resp, mustSend, renameSrc) } if filer_pb.IsDelete(resp) { return processDeleteEvent(filerSource, client, mountedDir, remoteStorageMountLocation, resp) @@ -210,6 +171,136 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem return eachEntryFunc, nil } +// reshapeCrossMountRename maps an update event onto the side of it the mount +// covers. The subscription delivers a rename when either of its paths lies +// under the mounted directory, and only a path under it maps to a remote +// key: toRemoteStorageLocation slices the mount prefix off, which panics on +// a shorter path and names a key in the wrong tree on a longer one. A rename +// into the mount is the creation of the new entry, a rename out of it the +// deletion of the old entry's object, unless that entry is remote-only and +// the object holds its only copy. An update with both paths inside is +// returned as is; one with neither is not the mount's (false). +func reshapeCrossMountRename(mountedDir string, resp *filer_pb.SubscribeMetadataResponse) (*filer_pb.SubscribeMetadataResponse, bool) { + message := resp.EventNotification + oldInMount := isUnderMount(mountedDir, resp.Directory, message.OldEntry.Name) + newInMount := isUnderMount(mountedDir, message.NewParentPath, message.NewEntry.Name) + switch { + case oldInMount && newInMount: + return resp, true + case newInMount: + glog.V(0).Infof("rename into mount %s: %s/%s -> %s/%s, creating", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return &filer_pb.SubscribeMetadataResponse{ + Directory: message.NewParentPath, + EventNotification: &filer_pb.EventNotification{NewEntry: message.NewEntry, NewParentPath: message.NewParentPath, Signatures: message.Signatures}, + TsNs: resp.TsNs, + }, true + case oldInMount: + if isRemoteOnly(message.OldEntry) { + glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, keeping the remote-only object", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return nil, false + } + glog.V(0).Infof("rename out of mount %s: %s/%s -> %s/%s, deleting", mountedDir, resp.Directory, message.OldEntry.Name, message.NewParentPath, message.NewEntry.Name) + return &filer_pb.SubscribeMetadataResponse{ + Directory: resp.Directory, + EventNotification: &filer_pb.EventNotification{OldEntry: message.OldEntry, Signatures: message.Signatures}, + TsNs: resp.TsNs, + }, true + default: + return nil, false + } +} + +// isUnderMount reports whether dir/name is the mounted directory or lies +// below it; a trailing slash on the mount flag does not change the answer. +func isUnderMount(mountedDir string, dir string, name string) bool { + mount := strings.TrimSuffix(mountedDir, "/") + path := string(util.NewFullPath(dir, name)) + return path == mount || strings.HasPrefix(path, mount+"/") +} + +// processCreateEvent uploads a created entry to the remote key its path maps +// to. mustSend skips the sync-stamp check: a create reshaped out of a rename +// into the mount inherits a RemoteEntry stamped at the old path's key, which +// says nothing about whether this destination holds the content. renameSrc is +// the path that rename came from, and is empty for a genuine create. +func processCreateEvent( + filerClient filer_pb.FilerClient, + filerSource filer_pb.FilerClient, + storageClass string, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + resp *filer_pb.SubscribeMetadataResponse, + mustSend bool, + renameSrc util.FullPath, +) error { + message := resp.EventNotification + if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { + return nil + } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } + // Propagate delete markers as deletions on the remote. + // Delete markers are zero-content version entries, so they + // would be filtered out by the HasData check below. + if isDeleteMarker(message.NewEntry) { + if newParent, newName, ok := rewriteVersionedSourcePath(message.NewParentPath, message.NewEntry.Name); ok { + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(newParent, newName), remoteStorageMountLocation) + return syncDeleteMarker(client, filerClient, message, dest) + } + return nil + } + // A create reshaped from a rename into the mount (mustSend) writes an empty + // entry as a zero-byte object; a dropped local copy still skips since the + // rename already copied its object to the new key. A stamped entry whose + // size diverged since (a truncate) is not a dropped copy and sends its + // new content. + if !filer.HasData(message.NewEntry) && !(mustSend && !isDroppedLocalCopy(message.NewEntry)) { + return nil + } + glog.V(2).Infof("create: %+v", resp) + if !mustSend && !shouldSendToRemote(message.NewEntry) { + glog.V(2).Infof("skipping creating: %+v", resp) + return nil + } + // Rewrite internal versioning paths to the original S3 key + // to prevent double-versioning when central also has versioning enabled + parentPath, entryName := message.NewParentPath, message.NewEntry.Name + isRewrittenVersion := false + if newParent, newName, ok := rewriteVersionedSourcePath(parentPath, entryName); ok { + glog.V(0).Infof("rewrite versioned path %s/%s -> %s/%s", parentPath, entryName, newParent, newName) + parentPath, entryName = newParent, newName + isRewrittenVersion = true + } + dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(parentPath, entryName), remoteStorageMountLocation) + if message.NewEntry.IsDirectory { + glog.V(0).Infof("mkdir %s", remote_storage.FormatLocation(dest)) + return client.WriteDirectory(dest, remoteWriteEntry(message.NewEntry, storageClass)) + } + remoteEntry, writeErr := retriedWriteFile(client, filerSource, message.NewParentPath, remoteWriteEntry(message.NewEntry, storageClass), dest) + if errors.Is(writeErr, errSuperseded) { + glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) + if mustSend { + return recoverSupersededCreate(filerClient, filerSource, client, mountedDir, remoteStorageMountLocation, message.NewParentPath, message.NewEntry.Name, renameSrc, dest, storageClass) + } + return nil + } + if writeErr != nil { + return writeErr + } + // Skip updateLocalEntry for versioned rewrites: the logical + // object (e.g. file.xml) has no filer entry in versioned + // buckets, and stamping the internal v_* entry with a + // RemoteEntry for the logical key is semantically wrong. + // Replay is safe because S3 PutObject is idempotent. + if isRewrittenVersion { + return nil + } + return updateLocalEntry(filerClient, message.NewParentPath, message.NewEntry, remoteEntry) +} + // processDeleteEvent removes the remote object an entry mapped to. A remote // object that is already absent counts as deleted: the entry was never // uploaded (created and deleted faster than the sync ran, or a replay of an @@ -651,6 +742,176 @@ func uploadCurrentEntry(filerClient filer_pb.FilerClient, filerSource filer_pb.F return updateLocalEntry(filerClient, dir, current, remoteEntry) } +// recoverSupersededCreate finishes a create reshaped from a rename into the +// mount when the filer has moved past the entry the event described. A plain +// superseded create skips because the superseding event follows in the log; a +// reshaped one has no later event for this destination. remote.uncache accepts +// the rename's inherited stamp and drops the chunks, and the update it logs +// skips the upload through isDroppedLocalCopy, so the content would exist +// nowhere. What the filer holds now decides: local content uploads here only +// when the superseding event will not (the stamp-edge case uploadCurrentEntry +// covers), and remote-only content copies from the object its stamp +// describes, the way completeRemoteOnlyRename finishes a rename. +func recoverSupersededCreate( + filerClient filer_pb.FilerClient, + filerSource filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + dir, name string, + renameSrc util.FullPath, + dest *remote_pb.RemoteStorageLocation, + storageClass string, +) error { + current, err := currentEntry(filerSource, dir, name) + if err != nil { + return err + } + if current == nil || current.IsDirectory { + return nil + } + if isRemoteOnly(current) { + return copyRenameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, dir, current, renameSrc, dest, storageClass) + } + if shouldSendToRemote(current) { + glog.V(0).Infof("leaving %s to the event that superseded the rename", remote_storage.FormatLocation(dest)) + return nil + } + glog.V(0).Infof("uploading the current %s in place of the superseded rename", remote_storage.FormatLocation(dest)) + remoteEntry, writeErr := retriedWriteFile(client, filerSource, dir, remoteWriteEntry(current, storageClass), dest) + if errors.Is(writeErr, errSuperseded) { + glog.Errorf("skipping %s: %v", remote_storage.FormatLocation(dest), writeErr) + return nil + } + if writeErr != nil { + return writeErr + } + return updateLocalEntry(filerClient, dir, current, remoteEntry) +} + +// copyRenameInSource gives the destination key the content of a remote-only +// entry a rename carried into the mount. The object sits at the key the +// renamed-from path maps to on its own mount's remote, or where a +// pending-rename marker under this mount points. The source object is left +// alone: the source side removes it through its own delete event. When the +// destination already holds what the stamp describes (an earlier run copied +// it), only the stamp is refreshed. With no object on either side the event +// fails and holds the offset. +func copyRenameInSource( + filerClient filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + dir string, + current *filer_pb.Entry, + renameSrc util.FullPath, + dest *remote_pb.RemoteStorageLocation, + storageClass string, +) error { + srcClient, srcLoc, err := renameInSource(filerClient, client, mountedDir, remoteStorageMountLocation, current, renameSrc) + if err != nil { + return err + } + if existing, err := client.StatFile(dest); err == nil { + if remote_storage.Describes(current.RemoteEntry, existing) { + glog.V(0).Infof("%s already holds the renamed content", remote_storage.FormatLocation(dest)) + return updateLocalEntry(filerClient, dir, current, existing) + } + glog.V(0).Infof("%s holds an object the entry does not describe (size %d, want %d); replacing it from %s", remote_storage.FormatLocation(dest), existing.RemoteSize, current.RemoteEntry.RemoteSize, remote_storage.FormatLocation(srcLoc)) + } else if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { + return err + } + stat, err := srcClient.StatFile(srcLoc) + if errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { + return fmt.Errorf("%s: content is on neither %s nor %s", util.NewFullPath(dir, current.Name), remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest)) + } + if err != nil { + return err + } + reader, err := openRemoteObject(srcClient, srcLoc, stat.RemoteSize) + if err != nil { + return err + } + defer reader.Close() + glog.V(0).Infof("copy %s -> %s", remote_storage.FormatLocation(srcLoc), remote_storage.FormatLocation(dest)) + remoteEntry, err := client.WriteFile(dest, remoteWriteEntry(current, storageClass), reader) + if err != nil { + return err + } + return updateLocalEntry(filerClient, dir, current, remoteEntry) +} + +// renameInSource locates the object holding a remote-only rename-in entry's +// content and the client that serves it. A pending-rename marker under this +// mount names a key on this remote; otherwise the renamed-from path is +// resolved through the filer's mount mappings to its own mount's remote +// location, served by this client when the source shares this storage. +func renameInSource( + filerClient filer_pb.FilerClient, + client remote_storage.RemoteStorageClient, + mountedDir string, + remoteStorageMountLocation *remote_pb.RemoteStorageLocation, + current *filer_pb.Entry, + renameSrc util.FullPath, +) (remote_storage.RemoteStorageClient, *remote_pb.RemoteStorageLocation, error) { + if src := util.FullPath(current.Extended[filer.ExtRemoteRenameSourceKey]); strings.HasPrefix(string(src), strings.TrimSuffix(mountedDir, "/")+"/") { + return client, toRemoteStorageLocation(util.FullPath(mountedDir), src, remoteStorageMountLocation), nil + } + mappings, err := readMountMappings(filerClient) + if err != nil { + return nil, nil, err + } + srcMount, srcRemoteLoc, err := filer.FindMountedRemoteMapping(mappings, string(renameSrc)) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + srcLoc := filer.MapFullPathToRemoteStorageLocation(util.FullPath(srcMount), srcRemoteLoc, renameSrc) + if srcRemoteLoc.Name == remoteStorageMountLocation.Name { + return client, srcLoc, nil + } + conf, err := readRemoteStorageConf(filerClient, srcRemoteLoc.Name) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + srcClient, err := remote_storage.GetRemoteStorage(conf) + if err != nil { + return nil, nil, fmt.Errorf("rename source %s: %w", renameSrc, err) + } + return srcClient, srcLoc, nil +} + +// readMountMappings reads the filer's remote mount table, the file +// DetectMountInfo reads at startup, through the event's filer client. +func readMountMappings(filerClient filer_pb.FilerClient) (mappings *remote_pb.RemoteStorageMapping, err error) { + var content []byte + err = filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + var readErr error + content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, filer.REMOTE_STORAGE_MOUNT_FILE) + return readErr + }) + if err != nil && !errors.Is(err, filer_pb.ErrNotFound) { + return nil, err + } + return filer.UnmarshalRemoteStorageMappings(content) +} + +// readRemoteStorageConf reads a remote storage's configuration entry. +func readRemoteStorageConf(filerClient filer_pb.FilerClient, storageName string) (*remote_pb.RemoteConf, error) { + var content []byte + if err := filerClient.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error { + var readErr error + content, readErr = filer.ReadInsideFiler(context.Background(), client, filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX) + return readErr + }); err != nil { + return nil, err + } + conf := &remote_pb.RemoteConf{} + if err := proto.Unmarshal(content, conf); err != nil { + return nil, fmt.Errorf("unmarshal %s/%s: %w", filer.DirectoryEtcRemote, storageName+filer.REMOTE_STORAGE_CONF_SUFFIX, err) + } + return conf, nil +} + // isSuperseded reports whether the filer has moved past the entry an event // described: it is deleted, or it no longer references every chunk the event // named. Those are the chunks the filer deletes when an entry is updated, so diff --git a/weed/command/filer_remote_sync_dir_test.go b/weed/command/filer_remote_sync_dir_test.go index 93c8340b47..5b004bd068 100644 --- a/weed/command/filer_remote_sync_dir_test.go +++ b/weed/command/filer_remote_sync_dir_test.go @@ -413,10 +413,12 @@ func TestIsMetadataOnlyUpdate(t *testing.T) { } // stubFilerClient serves one entry; nil means not found. A non-nil err fails -// every lookup instead. +// every lookup instead. entries, when set, answers lookups by "dir/name" +// before entry is consulted. type stubFilerClient struct { filer_pb.SeaweedFilerClient entry *filer_pb.Entry + entries map[string]*filer_pb.Entry err error updateErr error lookups int @@ -424,11 +426,14 @@ type stubFilerClient struct { updated *filer_pb.UpdateEntryRequest } -func (c *stubFilerClient) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest, ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { +func (c *stubFilerClient) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest, _ ...grpc.CallOption) (*filer_pb.LookupDirectoryEntryResponse, error) { c.lookups++ if c.err != nil { return nil, c.err } + if c.entries != nil { + return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entries[req.Directory+"/"+req.Name]}, nil + } return &filer_pb.LookupDirectoryEntryResponse{Entry: c.entry}, nil } @@ -2017,6 +2022,354 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) { } } +// TestReshapeCrossMountRename covers an update event with a path outside +// the mount. The subscription delivers a rename when either path is under +// the mount, and an update of the /etc/remote directory entry itself through +// the daemon's /etc/remote prefix; processUpdateEvent then sliced the mount +// prefix off both paths, which panics on a path shorter than the mount (seen +// on a replay: "slice bounds out of range [20:11]" on the 11-byte +// /etc/remote, every restart dying on the same event) and names a key in the +// wrong tree on a longer one. +func TestReshapeCrossMountRename(t *testing.T) { + const mount = "/buckets/mount" + chunked := func(name string) *filer_pb.Entry { + return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, Chunks: []*filer_pb.FileChunk{{FileId: "3,01", Size: 7}}} + } + event := func(oldDir, oldName, newDir, newName string, oldEntry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse { + if oldEntry == nil { + oldEntry = chunked(oldName) + } + return &filer_pb.SubscribeMetadataResponse{ + Directory: oldDir, + TsNs: 42, + EventNotification: &filer_pb.EventNotification{OldEntry: oldEntry, NewParentPath: newDir, NewEntry: chunked(newName), Signatures: []int32{7}}, + } + } + + t.Run("into the mount from a shorter path is a create of the new entry", func(t *testing.T) { + resp := event("/plain", "in.txt", mount+"/lock", "in.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsCreate(got) { + t.Fatalf("got %+v inMount=%v, want a create", got, inMount) + } + if got.Directory != mount+"/lock" || got.EventNotification.NewParentPath != mount+"/lock" || got.EventNotification.NewEntry.Name != "in.txt" { + t.Errorf("create at %s/%s (dir %s), want %s/in.txt", got.EventNotification.NewParentPath, got.EventNotification.NewEntry.Name, got.Directory, mount+"/lock") + } + if got.TsNs != 42 || len(got.EventNotification.Signatures) != 1 { + t.Errorf("timestamp and signatures must carry over: %+v", got) + } + }) + + t.Run("into the mount from a longer path outside it is a create, not a key in the wrong tree", func(t *testing.T) { + resp := event("/buckets/other/deeper/tree", "x.txt", mount, "x.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsCreate(got) || got.EventNotification.NewParentPath != mount { + t.Fatalf("got %+v inMount=%v, want a create under %s", got, inMount, mount) + } + }) + + t.Run("out of the mount is a delete of the old entry", func(t *testing.T) { + resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", nil) + got, inMount := reshapeCrossMountRename(mount, resp) + if !inMount || !filer_pb.IsDelete(got) { + t.Fatalf("got %+v inMount=%v, want a delete", got, inMount) + } + if got.Directory != mount+"/lock" || got.EventNotification.OldEntry.Name != "out.txt" { + t.Errorf("delete of %s/%s, want %s/out.txt", got.Directory, got.EventNotification.OldEntry.Name, mount+"/lock") + } + }) + + t.Run("out of the mount keeps the object of a remote-only entry", func(t *testing.T) { + remoteOnly := &filer_pb.Entry{Name: "out.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: &filer_pb.RemoteEntry{StorageName: "b2", RemoteSize: 7, RemoteMtime: 1786096669}} + resp := event(mount+"/lock", "out.txt", "/plain", "out.txt", remoteOnly) + if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("both paths inside is the update itself", func(t *testing.T) { + resp := event(mount+"/a", "x.txt", mount+"/b", "y.txt", nil) + if got, inMount := reshapeCrossMountRename(mount, resp); !inMount || got != resp { + t.Fatalf("got %+v inMount=%v, want the same event", got, inMount) + } + }) + + t.Run("an update of the /etc/remote directory entry itself is not the mount's event", func(t *testing.T) { + dirEntry := &filer_pb.Entry{Name: "remote", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + resp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/etc", + EventNotification: &filer_pb.EventNotification{OldEntry: dirEntry, NewParentPath: "/etc", NewEntry: dirEntry}, + } + if got, inMount := reshapeCrossMountRename("/buckets/c4x-storage", resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("neither path inside is not the mount's event", func(t *testing.T) { + resp := event("/plain", "x.txt", "/buckets/mounted-not", "x.txt", nil) + if got, inMount := reshapeCrossMountRename(mount, resp); inMount || got != nil { + t.Fatalf("got %+v inMount=%v, want the event dropped", got, inMount) + } + }) + + t.Run("a trailing slash on the mount flag changes nothing", func(t *testing.T) { + resp := event("/plain", "in.txt", mount, "in.txt", nil) + if got, inMount := reshapeCrossMountRename(mount+"/", resp); !inMount || !filer_pb.IsCreate(got) { + t.Fatalf("got %+v inMount=%v, want a create", got, inMount) + } + }) +} + +// TestRenameIntoMountUploadsDespiteTheOldStamp covers a file synced at a path +// under the mount, renamed out (which deleted its remote object), and renamed +// back in: the entry still carries the RemoteEntry the first sync stamped, so +// shouldSendToRemote would skip the create the reshape made, leaving the new +// key without an object. A reshaped create must send unconditionally. +func TestRenameIntoMountUploadsDespiteTheOldStamp(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"} + + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + newEntry := &filer_pb.Entry{ + Name: "back.txt", + Content: []byte("payload"), + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, + RemoteEntry: stamp, + } + resp, inMount := reshapeCrossMountRename(mountedDir, &filer_pb.SubscribeMetadataResponse{ + Directory: "/plain", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "back.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: stamp}, + NewParentPath: mountedDir + "/lock", + NewEntry: newEntry, + }, + }) + if !inMount || !filer_pb.IsCreate(resp) { + t.Fatalf("reshape = %+v inMount=%v, want a create", resp, inMount) + } + if shouldSendToRemote(resp.EventNotification.NewEntry) { + t.Fatal("precondition: the stamp the rename carried should fail shouldSendToRemote") + } + + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: newEntry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, true, "/plain/back.txt"); err != nil { + t.Fatal(err) + } + wantWrite := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/back.txt"} + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Fatalf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + + // a plain create keeps trusting the stamp + remote = &recordingRemote{} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, resp, false, ""); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the stamped create skipped", remote.writes) + } +} + +// TestRenameIntoMountWritesAnEmptyFile: a rename into the mount is reshaped +// to a create with mustSend, and an empty entry has no data for the HasData +// gate; the new key still takes a zero-byte object. A remote-only entry holds +// nothing local to send and is skipped as before. +func TestRenameIntoMountWritesAnEmptyFile(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/"} + renameIn := func(entry *filer_pb.Entry) *filer_pb.SubscribeMetadataResponse { + resp, inMount := reshapeCrossMountRename(mountedDir, &filer_pb.SubscribeMetadataResponse{ + Directory: "/plain", + EventNotification: &filer_pb.EventNotification{OldEntry: entry, NewParentPath: mountedDir + "/lock", NewEntry: entry}, + }) + if !inMount || !filer_pb.IsCreate(resp) { + t.Fatalf("reshape = %+v inMount=%v, want a create", resp, inMount) + } + return resp + } + wantWrite := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/lock/empty.txt"} + + t.Run("an empty entry writes a zero-byte object", func(t *testing.T) { + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + if filer.HasData(entry) { + t.Fatal("precondition: an empty entry carries no data") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Errorf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + }) + + t.Run("a remote-only entry still skips", func(t *testing.T) { + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 7}, RemoteEntry: stamp} + if !isRemoteOnly(entry) { + t.Fatal("precondition: the entry should be remote-only") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want none for a remote-only entry", remote.writes) + } + }) + + // a stamped entry truncated to zero is not remote-only content: its stamp + // describes bytes the file no longer holds, so the new key takes a + // zero-byte object + t.Run("a truncated stamped entry writes a zero-byte object", func(t *testing.T) { + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 7, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}, RemoteEntry: stamp} + if !isRemoteOnly(entry) || isDroppedLocalCopy(entry) { + t.Fatal("precondition: a truncated stamped entry reports remote-only but is not a dropped copy") + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), true, "/plain/empty.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], wantWrite) { + t.Errorf("writes = %+v, want the new key %s written", remote.writes, remote_storage.FormatLocation(wantWrite)) + } + }) + + // an ordinary create of an empty entry is unchanged: no object + t.Run("a plain empty create still writes nothing", func(t *testing.T) { + entry := &filer_pb.Entry{Name: "empty.txt", Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669}} + remote := &recordingRemote{} + filerClient := &stubFilerClient{entry: entry} + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn(entry), false, ""); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the empty create skipped", remote.writes) + } + }) +} + +// TestSupersededRenameInRecoversTheCurrentEntry covers a rename into the +// mount whose create event the sync reaches only after remote.uncache dropped +// the moved file's chunks: the upload is superseded, and skipping it leaves +// the destination object missing once the source side's delete runs. +func TestSupersededRenameInRecoversTheCurrentEntry(t *testing.T) { + const mountedDir = "/buckets/mount" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/mnt"} + srcMountLoc := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/plain-remote"} + + stamp := &filer_pb.RemoteEntry{StorageName: "b2", RemoteETag: "abc", RemoteSize: 4, RemoteMtime: 1786096669, LastLocalSyncTsNs: 1786096669*1e9 + 1} + // the entry as the rename logged it: local chunks plus the stamp earned + // under the source mount + eventEntry := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 4}, + Chunks: []*filer_pb.FileChunk{{FileId: "1,01111111", Size: 4}}, + RemoteEntry: stamp, + } + // the entry as remote.uncache left it: the chunks are gone and the stamp + // still describes the source mount's object + uncached := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096669, FileSize: 4}, + RemoteEntry: stamp, + } + mappingContent, err := proto.Marshal(&remote_pb.RemoteStorageMapping{Mappings: map[string]*remote_pb.RemoteStorageLocation{ + mountedDir: mountLoc, + "/plain": srcMountLoc, + }}) + if err != nil { + t.Fatal(err) + } + mappingEntry := &filer_pb.Entry{Name: filer.REMOTE_STORAGE_MOUNT_FILE, Content: mappingContent} + renameIn := &filer_pb.SubscribeMetadataResponse{ + Directory: mountedDir + "/lock", + EventNotification: &filer_pb.EventNotification{NewParentPath: mountedDir + "/lock", NewEntry: eventEntry}, + } + dest := &remote_pb.RemoteStorageLocation{Name: "b2", Bucket: "bucket", Path: "/mnt/lock/a.txt"} + + newFilerClient := func(entry *filer_pb.Entry) *stubFilerClient { + return &stubFilerClient{entries: map[string]*filer_pb.Entry{ + mountedDir + "/lock/a.txt": proto.Clone(entry).(*filer_pb.Entry), + filer.DirectoryEtcRemote + "/" + filer.REMOTE_STORAGE_MOUNT_FILE: mappingEntry, + }} + } + + t.Run("uncached meanwhile: the source object is copied and the entry stamped", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{"/plain-remote/a.txt": []byte("data")}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], dest) { + t.Fatalf("writes = %+v, want the source copied to %s", remote.writes, remote_storage.FormatLocation(dest)) + } + if string(remote.written[0]) != "data" { + t.Errorf("written = %q, want the source object's bytes", remote.written[0]) + } + if filerClient.updates != 1 { + t.Errorf("updates = %d, want the entry stamped with the copy", filerClient.updates) + } + }) + + t.Run("the destination already holds it: only a stamp", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{"/mnt/lock/a.txt": []byte("data")}, statETags: map[string]string{"/mnt/lock/a.txt": "abc"}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want none once the destination holds the content", remote.writes) + } + if filerClient.updates != 1 { + t.Errorf("updates = %d, want the entry stamped", filerClient.updates) + } + }) + + t.Run("no object on either side: the event fails", func(t *testing.T) { + remote := &recordingRemote{objects: map[string][]byte{}} + filerClient := newFilerClient(uncached) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err == nil { + t.Fatal("want the event to fail and hold the offset") + } + }) + + t.Run("a pending marker under the mount names the source", func(t *testing.T) { + marked := proto.Clone(uncached).(*filer_pb.Entry) + marked.Extended = map[string][]byte{filer.ExtRemoteRenameSourceKey: []byte(mountedDir + "/lock/old.txt")} + remote := &recordingRemote{objects: map[string][]byte{"/mnt/lock/old.txt": []byte("data")}} + filerClient := newFilerClient(marked) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 1 || !proto.Equal(remote.writes[0], dest) { + t.Fatalf("writes = %+v, want the marked source copied to %s", remote.writes, remote_storage.FormatLocation(dest)) + } + }) + + t.Run("rewritten since: the superseding event uploads", func(t *testing.T) { + rewritten := &filer_pb.Entry{ + Name: "a.txt", + Attributes: &filer_pb.FuseAttributes{Mtime: 1786096700, FileSize: 4}, + Chunks: []*filer_pb.FileChunk{{FileId: "2,02222222", Size: 4}}, + RemoteEntry: stamp, + } + remote := &recordingRemote{} + filerClient := newFilerClient(rewritten) + if err := processCreateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, renameIn, true, "/plain/a.txt"); err != nil { + t.Fatal(err) + } + if len(remote.writes) != 0 { + t.Errorf("writes = %+v, want the rewrite's own event to upload", remote.writes) + } + }) +} + func TestIsSystemPath(t *testing.T) { tests := []struct { dir string @@ -2086,4 +2439,3 @@ func TestRemoteSyncSkipsSystemFolder(t *testing.T) { t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes)) } } - From b86b2f3ceeff6495da6e6374e38667dfc0810c3b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Oct 2026 22:47:23 -0700 Subject: [PATCH 44/70] build(deps): bump github.com/go-chi/chi/v5 from 5.3.1 to 5.4.0 (#12004) Bumps [github.com/go-chi/chi/v5](https://github.com/go-chi/chi) from 5.3.1 to 5.4.0. - [Release notes](https://github.com/go-chi/chi/releases) - [Changelog](https://github.com/go-chi/chi/blob/master/CHANGELOG.md) - [Commits](https://github.com/go-chi/chi/compare/v5.3.1...v5.4.0) --- updated-dependencies: - dependency-name: github.com/go-chi/chi/v5 dependency-version: 5.4.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: user.email --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 73251d4c3c..ccfaa9ab5d 100644 --- a/go.mod +++ b/go.mod @@ -362,7 +362,7 @@ require ( github.com/flynn/noise v1.1.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/geoffgarside/ber v1.2.0 // indirect - github.com/go-chi/chi/v5 v5.3.1 // indirect + github.com/go-chi/chi/v5 v5.4.0 // indirect github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 // indirect github.com/go-jose/go-jose/v4 v4.1.5 // indirect github.com/go-logr/logr v1.4.4 // indirect diff --git a/go.sum b/go.sum index 4c1630efe7..a9f7da602a 100644 --- a/go.sum +++ b/go.sum @@ -483,8 +483,8 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= github.com/go-asn1-ber/asn1-ber v1.5.8 h1:H9AZkK22UOmfX8J84ubyaZxKJZ3FMHVwn8swoMML7iQ= github.com/go-asn1-ber/asn1-ber v1.5.8/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-chi/chi/v5 v5.3.1 h1:3j4HZLGZQ3JpMCrPJF/Jl3mYJfWLKBfNJ6quurUGCf8= -github.com/go-chi/chi/v5 v5.3.1/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto= +github.com/go-chi/chi/v5 v5.4.0 h1:+ufzvFTGKyin26Or5DLrlIDq4ygIDBVcj7oVNeeaYFE= +github.com/go-chi/chi/v5 v5.4.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68= github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo= github.com/go-errors/errors v1.5.1 h1:ZwEMSLRCapFLflTpT7NKaAc7ukJ8ZPEjzlxt8rPN8bk= From 6a34d82f05aa8d22eb00d24542ac4f9dae8e00e4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 22:52:51 -0700 Subject: [PATCH 45/70] shell: retry filer discovery until master knows a filer (#11989) * shell: discover the filer on first use instead of at startup The startup retry loop made every weed shell invocation wait out the connect timeout on clusters that have no filer, even for commands that only talk to the master. Discovery now runs once, on the first command that actually needs a filer, keeping the retry-until-registered fix for the fresh-cluster race without stalling master-only commands. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * shell: stop filer discovery retries on non-transient errors A master that keeps answering ListClusterNodes with a permanent error (permission denied, unimplemented) cannot gain a filer by retrying, so discovery returns instead of polling until the deadline. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * shell: explain an empty filer list when discovery times out * shell: retry filer discovery while the address is still empty Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/shell/command_fs_configure.go | 2 +- weed/shell/command_fs_meta_save.go | 4 +- weed/shell/command_fs_verify.go | 2 +- weed/shell/command_mq_topic_compact.go | 2 +- weed/shell/command_remote_meta_sync.go | 2 +- weed/shell/command_remote_mount.go | 4 +- weed/shell/command_remote_mount_buckets.go | 2 +- weed/shell/command_remote_uncache.go | 2 +- weed/shell/command_remote_unmount.go | 4 +- weed/shell/command_s3_bucket_quota_check.go | 2 +- weed/shell/command_s3_clean_uploads.go | 2 +- weed/shell/command_s3_iam_client.go | 2 +- weed/shell/command_volume_fsck.go | 2 +- weed/shell/commands.go | 71 +++++- weed/shell/commands_test.go | 159 +++++++++++++ weed/shell/s3tables_helpers.go | 2 +- weed/shell/shell_liner.go | 66 ------ weed/shell/shell_liner_test.go | 241 -------------------- 18 files changed, 246 insertions(+), 325 deletions(-) create mode 100644 weed/shell/commands_test.go diff --git a/weed/shell/command_fs_configure.go b/weed/shell/command_fs_configure.go index 4bc29e3984..165e0b8e65 100644 --- a/weed/shell/command_fs_configure.go +++ b/weed/shell/command_fs_configure.go @@ -83,7 +83,7 @@ func (c *commandFsConfigure) Do(args []string, commandEnv *CommandEnv, writer io return nil } - fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) + fc, err := filer.ReadFilerConf(commandEnv.FilerAddress(), commandEnv.option.GrpcDialOption, commandEnv.MasterClient) if err != nil { return err } diff --git a/weed/shell/command_fs_meta_save.go b/weed/shell/command_fs_meta_save.go index 4034655350..c52cc421dc 100644 --- a/weed/shell/command_fs_meta_save.go +++ b/weed/shell/command_fs_meta_save.go @@ -75,7 +75,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. if fileName == "" { t := time.Now() fileName = fmt.Sprintf("%s-%4d%02d%02d-%02d%02d%02d.meta.gz", - commandEnv.option.FilerAddress.ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) + commandEnv.FilerAddress().ToHttpAddress(), t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second()) } var dst io.Writer @@ -143,7 +143,7 @@ func (c *commandFsMetaSave) Do(args []string, commandEnv *CommandEnv, writer io. }) if err == nil { - fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.option.FilerAddress.ToHttpAddress(), path, fileName) + fmt.Fprintf(writer, "meta data for http://%s%s is saved to %s\n", commandEnv.FilerAddress().ToHttpAddress(), path, fileName) } return err diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.go index 32128b4b6f..24007793f0 100644 --- a/weed/shell/command_fs_verify.go +++ b/weed/shell/command_fs_verify.go @@ -254,7 +254,7 @@ func (c *commandFsVerify) verifyProcessMetadata(path string, wg *sync.WaitGroup) StopTsNs: time.Now().UnixNano(), EventErrorType: pb.DontLogError, } - err = pb.FollowMetadata(c.env.option.FilerAddress, c.env.option.GrpcDialOption, metadataFollowOption, processEventFn) + err = pb.FollowMetadata(c.env.FilerAddress(), c.env.option.GrpcDialOption, metadataFollowOption, processEventFn) return fileCount, errCount, prunedCount, err } diff --git a/weed/shell/command_mq_topic_compact.go b/weed/shell/command_mq_topic_compact.go index ab3552022c..8171144b1f 100644 --- a/weed/shell/command_mq_topic_compact.go +++ b/weed/shell/command_mq_topic_compact.go @@ -73,7 +73,7 @@ func (c *commandMqTopicCompact) Do(args []string, commandEnv *CommandEnv, writer // read topic configuration fca := filer_client.NewFilerClientAccessor( - []pb.ServerAddress{commandEnv.option.FilerAddress}, + []pb.ServerAddress{commandEnv.FilerAddress()}, commandEnv.option.GrpcDialOption, ) t := topic.NewTopic(*namespace, *topicName) diff --git a/weed/shell/command_remote_meta_sync.go b/weed/shell/command_remote_meta_sync.go index b1ce83140b..ca9320d3f5 100644 --- a/weed/shell/command_remote_meta_sync.go +++ b/weed/shell/command_remote_meta_sync.go @@ -84,7 +84,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer } func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) { - return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, dir) + return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), dir) } /* diff --git a/weed/shell/command_remote_mount.go b/weed/shell/command_remote_mount.go index 1eeb43bf53..debfa27a63 100644 --- a/weed/shell/command_remote_mount.go +++ b/weed/shell/command_remote_mount.go @@ -85,7 +85,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io } // find configuration for remote storage - remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, remote_storage.ParseLocationName(*remote)) + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), remote_storage.ParseLocationName(*remote)) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } @@ -133,7 +133,7 @@ func (c *commandRemoteMount) Do(args []string, commandEnv *CommandEnv, writer io func listExistingRemoteStorageMounts(commandEnv *CommandEnv, writer io.Writer) (mappings *remote_pb.RemoteStorageMapping, err error) { // read current mapping - mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, err = filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if err != nil { return mappings, err } diff --git a/weed/shell/command_remote_mount_buckets.go b/weed/shell/command_remote_mount_buckets.go index 1fa464141c..c011a15e57 100644 --- a/weed/shell/command_remote_mount_buckets.go +++ b/weed/shell/command_remote_mount_buckets.go @@ -62,7 +62,7 @@ func (c *commandRemoteMountBuckets) Do(args []string, commandEnv *CommandEnv, wr } // find configuration for remote storage - remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *remote) + remoteConf, err := filer.ReadRemoteStorageConf(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *remote) if err != nil { return fmt.Errorf("find configuration for %s: %v", *remote, err) } diff --git a/weed/shell/command_remote_uncache.go b/weed/shell/command_remote_uncache.go index e7281c791f..20383a8f40 100644 --- a/weed/shell/command_remote_uncache.go +++ b/weed/shell/command_remote_uncache.go @@ -57,7 +57,7 @@ func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer return nil } - mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if listErr != nil { return listErr } diff --git a/weed/shell/command_remote_unmount.go b/weed/shell/command_remote_unmount.go index 030dd9021a..fd905d00ee 100644 --- a/weed/shell/command_remote_unmount.go +++ b/weed/shell/command_remote_unmount.go @@ -52,7 +52,7 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer return nil } - mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress) + mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress()) if listErr != nil { return listErr } @@ -79,7 +79,7 @@ func (c *commandRemoteUnmount) Do(args []string, commandEnv *CommandEnv, writer } // reset remote sync offset in case the folder is mounted again - if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.option.FilerAddress, *dir, time.Now().UnixNano()); err != nil { + if err = remote_storage.SetSyncOffset(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), *dir, time.Now().UnixNano()); err != nil { return fmt.Errorf("reset remote.sync offset for %s: %v", *dir, err) } diff --git a/weed/shell/command_s3_bucket_quota_check.go b/weed/shell/command_s3_bucket_quota_check.go index 7313514ec2..eda0c42842 100644 --- a/weed/shell/command_s3_bucket_quota_check.go +++ b/weed/shell/command_s3_bucket_quota_check.go @@ -60,7 +60,7 @@ func (c *commandS3BucketQuotaEnforce) Do(args []string, commandEnv *CommandEnv, } // read existing filer configuration - fc, err := filer.ReadFilerConf(commandEnv.option.FilerAddress, commandEnv.option.GrpcDialOption, commandEnv.MasterClient) + fc, err := filer.ReadFilerConf(commandEnv.FilerAddress(), commandEnv.option.GrpcDialOption, commandEnv.MasterClient) if err != nil { return err } diff --git a/weed/shell/command_s3_clean_uploads.go b/weed/shell/command_s3_clean_uploads.go index 1da7a02d40..1d7bf986c0 100644 --- a/weed/shell/command_s3_clean_uploads.go +++ b/weed/shell/command_s3_clean_uploads.go @@ -106,7 +106,7 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io fmt.Fprintf(writer, "skip %s: %v\n", staleUpload.Name, checkErr) continue } - deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload.Name) + deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.FilerAddress().ToHttpAddress(), uploadsDir, staleUpload.Name) if completed { deleteUrl += "&skipChunkDeletion=true" } diff --git a/weed/shell/command_s3_iam_client.go b/weed/shell/command_s3_iam_client.go index eb619100a1..832471d0bf 100644 --- a/weed/shell/command_s3_iam_client.go +++ b/weed/shell/command_s3_iam_client.go @@ -28,7 +28,7 @@ func (ce *CommandEnv) withIamClient(fn func(ctx context.Context, client iam_pb.S ctx, cancel := context.WithTimeout(iamAdminAuthContext(context.Background()), iamRequestTimeout) defer cancel() return fn(ctx, iam_pb.NewSeaweedIdentityAccessManagementClient(conn)) - }, ce.option.FilerAddress.ToGrpcAddress(), false, ce.option.GrpcDialOption) + }, ce.FilerAddress().ToGrpcAddress(), false, ce.option.GrpcDialOption) } func iamAdminAuthContext(ctx context.Context) context.Context { diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index e3c301992d..ef6b61bfab 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -790,7 +790,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) { req.URL = &url.URL{ Scheme: "http", - Host: c.env.option.FilerAddress.ToHttpAddress(), + Host: c.env.FilerAddress().ToHttpAddress(), Path: string(path), } diff --git a/weed/shell/commands.go b/weed/shell/commands.go index 3bbb363b18..43b681d4d7 100644 --- a/weed/shell/commands.go +++ b/weed/shell/commands.go @@ -4,7 +4,10 @@ import ( "context" "fmt" "io" + "math/rand/v2" + "os" "strings" + "sync" "time" "github.com/seaweedfs/seaweedfs/weed/cluster" @@ -43,6 +46,7 @@ type CommandEnv struct { noLock bool forceNoLock bool verbose bool + filerMu sync.Mutex } func NewCommandEnv(options *ShellOptions) *CommandEnv { @@ -147,10 +151,75 @@ var _ = filer_pb.FilerClient(&CommandEnv{}) func (ce *CommandEnv) WithFilerClient(streamingMode bool, fn func(filer_pb.SeaweedFilerClient) error) error { - return pb.WithGrpcFilerClient(streamingMode, 0, ce.option.FilerAddress, ce.option.GrpcDialOption, fn) + return pb.WithGrpcFilerClient(streamingMode, 0, ce.FilerAddress(), ce.option.GrpcDialOption, fn) } +// FilerAddress returns the discovered or configured filer, running discovery +// on first use so commands that never need a filer skip the wait entirely. +// A failed or empty discovery is not cached: each later call retries until a +// filer registers, so a long-lived shell can still find one. +func (ce *CommandEnv) FilerAddress() pb.ServerAddress { + ce.filerMu.Lock() + defer ce.filerMu.Unlock() + ce.discoverFiler() + return ce.option.FilerAddress +} + +func (ce *CommandEnv) discoverFiler() { + if ce.option.FilerAddress != "" || ce.MasterClient == nil || ce.option.FilerGroup == nil { + return + } + connectTimeout := ce.option.ConnectTimeout + if connectTimeout <= 0 { + connectTimeout = 30 * time.Second + } + ctx, cancel := context.WithTimeout(context.Background(), connectTimeout) + defer cancel() + var filers []pb.ServerAddress + var listErr error + for len(filers) == 0 && ctx.Err() == nil { + filers = filers[:0] + listErr = ce.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { + resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ + ClientType: cluster.FilerType, + FilerGroup: *ce.option.FilerGroup, + }) + if err != nil { + return err + } + + for _, clusterNode := range resp.ClusterNodes { + filers = append(filers, pb.ServerAddress(clusterNode.Address)) + } + return nil + }) + if len(filers) == 0 { + if listErr != nil && !util.IsTransientError(listErr) { + break + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + } + if len(filers) > 0 { + ce.option.FilerAddress = filers[rand.IntN(len(filers))] + } else if listErr != nil { + fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *ce.option.Masters, listErr) + } else { + fmt.Fprintf(os.Stderr, "no filer registered with master %s within %s\n", *ce.option.Masters, connectTimeout) + } + if ce.option.Debug { + if len(filers) > 0 { + fmt.Fprintf(os.Stderr, "master: %s filers: %v\n", *ce.option.Masters, filers) + } else { + fmt.Fprintf(os.Stderr, "master: %s\n", *ce.option.Masters) + } + } +} + func (ce *CommandEnv) AdjustedUrl(location *filer_pb.Location) string { return location.Url } diff --git a/weed/shell/commands_test.go b/weed/shell/commands_test.go new file mode 100644 index 0000000000..8456413b7a --- /dev/null +++ b/weed/shell/commands_test.go @@ -0,0 +1,159 @@ +package shell + +import ( + "context" + "fmt" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" +) + +type fakeDiscoveryServer struct { + master_pb.UnimplementedSeaweedServer + leader string + filers atomic.Int32 + listErr atomic.Value +} + +func (s *fakeDiscoveryServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&master_pb.KeepConnectedResponse{ + VolumeLocation: &master_pb.VolumeLocation{Leader: s.leader}, + }); err != nil { + return err + } + <-stream.Context().Done() + return nil +} + +func (s *fakeDiscoveryServer) ListClusterNodes(context.Context, *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) { + if err, _ := s.listErr.Load().(error); err != nil { + return nil, err + } + resp := &master_pb.ListClusterNodesResponse{} + for i := int32(0); i < s.filers.Load(); i++ { + resp.ClusterNodes = append(resp.ClusterNodes, &master_pb.ListClusterNodesResponse_ClusterNode{ + Address: fmt.Sprintf("127.0.0.1:88%02d", i), + }) + } + return resp, nil +} + +func startFakeDiscoveryMaster(t *testing.T, srv *fakeDiscoveryServer) string { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, srv) + go func() { _ = grpcServer.Serve(lis) }() + t.Cleanup(grpcServer.GracefulStop) + + _, port, _ := net.SplitHostPort(lis.Addr().String()) + return fmt.Sprintf("127.0.0.1:0.%s", port) +} + +func discoveryEnv(addr string, timeout time.Duration) (*CommandEnv, context.CancelFunc) { + masters := addr + filerGroup := "" + options := &ShellOptions{ + Masters: &masters, + GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), + FilerGroup: &filerGroup, + Directory: "/", + ConnectTimeout: timeout, + } + ce := NewCommandEnv(options) + ctx, cancel := context.WithCancel(context.Background()) + go ce.MasterClient.KeepConnectedToMaster(ctx) + ce.MasterClient.WaitUntilConnected(ctx) + return ce, cancel +} + +func TestFilerDiscoveryRetriesUntilFilerAppears(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 10*time.Second) + defer cancel() + + go func() { + time.Sleep(1200 * time.Millisecond) + srv.filers.Store(1) + }() + + got := ce.FilerAddress() + if got != "127.0.0.1:8800" { + t.Fatalf("expected the late-registering filer, got %q", got) + } +} + +func TestFilerDiscoveryReturnsEmptyAtDeadline(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 1500*time.Millisecond) + defer cancel() + + start := time.Now() + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + if elapsed := time.Since(start); elapsed < time.Second { + t.Fatalf("discovery returned after %v, expected it to poll until the deadline", elapsed) + } +} + +func TestFilerDiscoveryRetriesAfterEmptyResult(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + + ce, cancel := discoveryEnv(addr, 1500*time.Millisecond) + defer cancel() + + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + srv.filers.Store(1) + if got := ce.FilerAddress(); got != "127.0.0.1:8800" { + t.Fatalf("expected the late-registered filer on retry, got %q", got) + } +} + +func TestFilerDiscoveryStopsOnNonTransientError(t *testing.T) { + srv := &fakeDiscoveryServer{} + addr := startFakeDiscoveryMaster(t, srv) + srv.leader = addr + srv.listErr.Store(status.Error(codes.PermissionDenied, "denied")) + + ce, cancel := discoveryEnv(addr, 30*time.Second) + defer cancel() + + start := time.Now() + if got := ce.FilerAddress(); got != "" { + t.Fatalf("expected no filer, got %q", got) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Fatalf("discovery kept polling a non-transient error for %v", elapsed) + } +} + +func TestFilerAddressConfiguredSkipsDiscovery(t *testing.T) { + ce := &CommandEnv{option: &ShellOptions{FilerAddress: "127.0.0.1:8888"}} + if got := ce.FilerAddress(); got != "127.0.0.1:8888" { + t.Fatalf("expected the configured filer, got %q", got) + } +} diff --git a/weed/shell/s3tables_helpers.go b/weed/shell/s3tables_helpers.go index d595a5af74..18c5f90a72 100644 --- a/weed/shell/s3tables_helpers.go +++ b/weed/shell/s3tables_helpers.go @@ -20,7 +20,7 @@ func withFilerClient(commandEnv *CommandEnv, fn func(client filer_pb.SeaweedFile return pb.WithGrpcClient(context.Background(), false, 0, func(conn *grpc.ClientConn) error { client := filer_pb.NewSeaweedFilerClient(conn) return fn(client) - }, commandEnv.option.FilerAddress.ToGrpcAddress(), false, commandEnv.option.GrpcDialOption) + }, commandEnv.FilerAddress().ToGrpcAddress(), false, commandEnv.option.GrpcDialOption) } func executeS3Tables(commandEnv *CommandEnv, operation string, req interface{}, resp interface{}, accountID string) error { diff --git a/weed/shell/shell_liner.go b/weed/shell/shell_liner.go index 87d18c9b6c..303f2ab2ad 100644 --- a/weed/shell/shell_liner.go +++ b/weed/shell/shell_liner.go @@ -3,19 +3,14 @@ package shell import ( "bufio" "context" - "errors" "fmt" "io" - "math/rand/v2" "os" "path" "slices" "strings" "time" - "github.com/seaweedfs/seaweedfs/weed/cluster" - "github.com/seaweedfs/seaweedfs/weed/pb" - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util/grace" @@ -26,47 +21,6 @@ import ( var historyPath = path.Join(os.TempDir(), "weed-shell") -// errNoFilersDiscovered marks a successful ListClusterNodes call that came -// back empty, which usually means the filer has not registered with the -// master yet. Unlike an RPC failure it is expected at startup, so filer -// discovery retries on it until the connect timeout expires. -var errNoFilersDiscovered = errors.New("no filer registered with master yet") - -// listFilers polls ListClusterNodes until at least one filer is registered -// with the master or ctx expires. An empty list is expected while a filer is -// still starting, so it retries like a transient failure. On return, a nil -// error with an empty list means the master answered but no filer registered -// in time; a non-nil error is the last attempt's failure. -func listFilers(ctx context.Context, commandEnv *CommandEnv, filerGroup string, maxWait time.Duration) (filers []pb.ServerAddress, listErr error) { - util.RetryWithBackoff(ctx, "filer discovery", maxWait, - func(err error) bool { - return errors.Is(err, errNoFilersDiscovered) || util.IsTransientError(err) - }, func() error { - filers = filers[:0] - listErr = commandEnv.MasterClient.WithClient(ctx, false, func(client master_pb.SeaweedClient) error { - resp, err := client.ListClusterNodes(ctx, &master_pb.ListClusterNodesRequest{ - ClientType: cluster.FilerType, - FilerGroup: filerGroup, - }) - if err != nil { - return err - } - for _, clusterNode := range resp.ClusterNodes { - filers = append(filers, pb.ServerAddress(clusterNode.Address)) - } - return nil - }) - if listErr != nil { - return listErr - } - if len(filers) == 0 { - return errNoFilersDiscovered - } - return nil - }) - return filers, listErr -} - // Piped stdin returns the last command failure; an interactive session shows // the error to the operator and keeps going. func RunShell(options ShellOptions) error { @@ -116,26 +70,6 @@ func RunShell(options ShellOptions) error { return err } - if commandEnv.option.FilerAddress == "" { - discoverCtx, cancelDiscover := context.WithTimeout(ctx, connectTimeout) - defer cancelDiscover() - filers, listErr := listFilers(discoverCtx, commandEnv, *options.FilerGroup, connectTimeout) - if len(filers) > 0 { - commandEnv.option.FilerAddress = filers[rand.IntN(len(filers))] - } else if listErr != nil { - fmt.Fprintf(os.Stderr, "cannot list filers from master %s: %v\n", *options.Masters, listErr) - } else { - fmt.Fprintf(os.Stderr, "no filer registered with master %s within %s\n", *options.Masters, connectTimeout) - } - if options.Debug { - if len(filers) > 0 { - fmt.Fprintf(os.Stderr, "master: %s filers: %v\n", *options.Masters, filers) - } else { - fmt.Fprintf(os.Stderr, "master: %s\n", *options.Masters) - } - } - } - if interactive { for { cmd, err := line.Prompt("> ") diff --git a/weed/shell/shell_liner_test.go b/weed/shell/shell_liner_test.go index d8268f887f..21a707ac8a 100644 --- a/weed/shell/shell_liner_test.go +++ b/weed/shell/shell_liner_test.go @@ -1,22 +1,11 @@ package shell import ( - "context" "errors" - "fmt" "io" - "net" - "os" "strings" - "sync/atomic" "testing" "time" - - "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/status" ) type fakeCommand struct { @@ -94,233 +83,3 @@ func TestProcessEachCmdPreservesRegisteredCommandError(t *testing.T) { t.Errorf("a succeeding command must return no error, got exit=%v err=%v", exit, err) } } - -// fakeMaster implements just enough of master_pb.SeaweedServer for filer -// discovery: KeepConnected holds the stream open so the client marks the -// master connected, and ListClusterNodes answers from a script. -type fakeMaster struct { - master_pb.UnimplementedSeaweedServer - listCalls atomic.Int32 - onList func() (*master_pb.ListClusterNodesResponse, error) -} - -func (f *fakeMaster) KeepConnected(stream grpc.BidiStreamingServer[master_pb.KeepConnectedRequest, master_pb.KeepConnectedResponse]) error { - for { - if _, err := stream.Recv(); err != nil { - return err - } - } -} - -func (f *fakeMaster) ListClusterNodes(context.Context, *master_pb.ListClusterNodesRequest) (*master_pb.ListClusterNodesResponse, error) { - f.listCalls.Add(1) - return f.onList() -} - -func (f *fakeMaster) VolumeList(context.Context, *master_pb.VolumeListRequest) (*master_pb.VolumeListResponse, error) { - return &master_pb.VolumeListResponse{TopologyInfo: &master_pb.TopologyInfo{}}, nil -} - -// startFakeMaster serves the fake master on an ephemeral gRPC port and -// returns the http-form master address (whose gRPC port is port+10000) to -// hand to ShellOptions.Masters. -func startFakeMaster(t *testing.T) (*fakeMaster, string) { - t.Helper() - listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("fake master listen: %v", err) - } - grpcPort := listener.Addr().(*net.TCPAddr).Port - if grpcPort < 10001 { - listener.Close() - t.Fatalf("fake master needs a port above 10000 for its http form, got %d", grpcPort) - } - fake := &fakeMaster{} - server := grpc.NewServer() - master_pb.RegisterSeaweedServer(server, fake) - go server.Serve(listener) - t.Cleanup(server.Stop) - return fake, fmt.Sprintf("127.0.0.1:%d", grpcPort-10000) -} - -func newTestCommandEnv(t *testing.T, masters string) *CommandEnv { - t.Helper() - filerGroup := "" - options := &ShellOptions{ - Masters: &masters, - FilerGroup: &filerGroup, - GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), - } - commandEnv := NewCommandEnv(options) - ctx, cancel := context.WithCancel(context.Background()) - go commandEnv.MasterClient.KeepConnectedToMaster(ctx) - t.Cleanup(cancel) - return commandEnv -} - -func filerNode(address string) *master_pb.ListClusterNodesResponse { - return &master_pb.ListClusterNodesResponse{ - ClusterNodes: []*master_pb.ListClusterNodesResponse_ClusterNode{ - {Address: address}, - }, - } -} - -// A filer registering mid-discovery turns an empty list into a populated one -// on a later attempt. -func TestListFilersEmptyThenPopulated(t *testing.T) { - master, masters := startFakeMaster(t) - master.onList = func() (*master_pb.ListClusterNodesResponse, error) { - if master.listCalls.Load() <= 2 { - return &master_pb.ListClusterNodesResponse{}, nil - } - return filerNode("127.0.0.1:9888"), nil - } - - commandEnv := newTestCommandEnv(t, masters) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - filers, err := listFilers(ctx, commandEnv, "", 10*time.Second) - if err != nil { - t.Fatalf("listFilers failed: %v", err) - } - if len(filers) != 1 || filers[0] != "127.0.0.1:9888" { - t.Fatalf("expected the late-registered filer, got %v", filers) - } - if got := master.listCalls.Load(); got != 3 { - t.Fatalf("expected 3 discovery attempts, got %d", got) - } -} - -// A master that answers but never lists a filer exhausts the discovery -// window and reports an empty, error-free result — the caller then prints -// the "no filer registered" notice and continues without a filer. -func TestListFilersEmptyUntilTimeout(t *testing.T) { - master, masters := startFakeMaster(t) - master.onList = func() (*master_pb.ListClusterNodesResponse, error) { - return &master_pb.ListClusterNodesResponse{}, nil - } - commandEnv := newTestCommandEnv(t, masters) - - start := time.Now() - ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) - defer cancel() - - filers, err := listFilers(ctx, commandEnv, "", 1500*time.Millisecond) - if err != nil { - t.Fatalf("empty discovery must not return an error, got: %v", err) - } - if len(filers) != 0 { - t.Fatalf("expected no filers, got %v", filers) - } - if got := master.listCalls.Load(); got < 2 { - t.Fatalf("expected repeated polling, got %d attempts", got) - } - if elapsed := time.Since(start); elapsed > 10*time.Second { - t.Fatalf("discovery should give up near the timeout, took %v", elapsed) - } -} - -// A non-transient failure on the last attempt surfaces as the discovery -// error, so the diagnostic reports the RPC failure instead of a missing -// filer. -func TestListFilersReportsLastError(t *testing.T) { - master, masters := startFakeMaster(t) - master.onList = func() (*master_pb.ListClusterNodesResponse, error) { - return nil, status.Error(codes.PermissionDenied, "denied") - } - commandEnv := newTestCommandEnv(t, masters) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - filers, err := listFilers(ctx, commandEnv, "", 10*time.Second) - if err == nil { - t.Fatalf("expected the permission error to be reported") - } - if len(filers) != 0 { - t.Fatalf("expected no filers, got %v", filers) - } - if got := master.listCalls.Load(); got != 1 { - t.Fatalf("a non-transient error must stop retries, got %d attempts", got) - } -} - -// A transient RPC failure on the last attempt also surfaces as the discovery -// error rather than as a missing filer. -func TestListFilersErrorAfterEmptyList(t *testing.T) { - master, masters := startFakeMaster(t) - master.onList = func() (*master_pb.ListClusterNodesResponse, error) { - if master.listCalls.Load() <= 1 { - return &master_pb.ListClusterNodesResponse{}, nil - } - return nil, status.Error(codes.Unavailable, "master gone") - } - commandEnv := newTestCommandEnv(t, masters) - - ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) - defer cancel() - - filers, err := listFilers(ctx, commandEnv, "", 1500*time.Millisecond) - if err == nil { - t.Fatalf("expected the late RPC failure to be reported") - } - if len(filers) != 0 { - t.Fatalf("expected no filers, got %v", filers) - } -} - -// After discovery times out with no filer, master-only commands still run: -// the shell warns about the missing filer but proceeds to execute stdin. -func TestRunShellFilerDiscoveryTimeout(t *testing.T) { - master, masters := startFakeMaster(t) - master.onList = func() (*master_pb.ListClusterNodesResponse, error) { - return &master_pb.ListClusterNodesResponse{}, nil - } - filerGroup := "" - - oldStdin := os.Stdin - inReader, inWriter, err := os.Pipe() - if err != nil { - t.Fatalf("stdin pipe: %v", err) - } - os.Stdin = inReader - t.Cleanup(func() { os.Stdin = oldStdin }) - go func() { - fmt.Fprintln(inWriter, "cluster.ps") - fmt.Fprintln(inWriter, "exit") - inWriter.Close() - }() - - oldStderr := os.Stderr - errReader, errWriter, err := os.Pipe() - if err != nil { - t.Fatalf("stderr pipe: %v", err) - } - os.Stderr = errWriter - t.Cleanup(func() { os.Stderr = oldStderr }) - - done := make(chan error, 1) - go func() { - done <- RunShell(ShellOptions{ - Masters: &masters, - FilerGroup: &filerGroup, - GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()), - ConnectTimeout: 1500 * time.Millisecond, - }) - }() - - select { - case err = <-done: - case <-time.After(15 * time.Second): - t.Fatalf("RunShell did not return after discovery timeout") - } - errWriter.Close() - stderr, _ := io.ReadAll(errReader) - if err != nil { - t.Fatalf("master-only commands must run without a filer, got: %v", err) - } - if !strings.Contains(string(stderr), "no filer registered") { - t.Fatalf("expected the missing-filer notice on stderr, got: %q", stderr) - } -} From ecd778a4a0c62790c51b9352033d49a6597d22ba Mon Sep 17 00:00:00 2001 From: Javier de la Torre Date: Sat, 24 Oct 2026 07:53:43 +0200 Subject: [PATCH 46/70] iceberg maintenance: a merged file with null row ids names no first_row_id (#11973) * iceberg maintenance: a merged file with null row ids names no first_row_id Compaction stamps a merged v3 data file with the _row_id of its first row as first_row_id (mergedFileFirstRowID). When the bin mixed rows that carry a materialized _row_id with rows that carry none (a data file with no first_row_id, as a v2 table's files are after an upgrade to v3, gives null ids), the null rows then inherit first_row_id + their position in the file: ids that other rows already hold, so two rows answer to the same row id. A merged file now names a first_row_id only when its footer shows no null _row_id in any row group. Otherwise the commit assigns the file a fresh range from the table's next-row-id: the null rows get new, unused ids and the materialized ones keep theirs (a materialized value overrides inheritance). Test: TestMergedFileFirstRowIDOnlyWithoutNullIDs ("materialized first, a null later" fails without the fix). * iceberg maintenance: name no first_row_id when the merged file has a null _row_id mergedFileFirstRowID stamped the merged file with its first row's id without checking later rows. A null _row_id in a merged file inherits first_row_id+position, an id another row already holds, so the commit produced two rows answering to the same row id. Check the footer's null counts for the _row_id column across row groups first: any null means the file is committed without a first_row_id, so its nulls take fresh ids from the table's next-row-id while materialized values still override inheritance. * shell: retry filer discovery until master knows a filer A shell started against a freshly booted cluster (weed mini, the s3tables/lance test harnesses) can list cluster nodes before the filer's first heartbeat lands on the master. The empty list left FilerAddress unset and the next command died with "dial tcp: missing address". Poll ListClusterNodes until a filer appears or the connect timeout expires. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * shell: discover the filer on first use instead of at startup The startup retry loop made every weed shell invocation wait out the connect timeout on clusters that have no filer, even for commands that only talk to the master. Discovery now runs once, on the first command that actually needs a filer, keeping the retry-until-registered fix for the fresh-cluster race without stalling master-only commands. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * shell: stop filer discovery retries on non-transient errors A master that keeps answering ListClusterNodes with a permanent error (permission denied, unimplemented) cannot gain a filer by retrying, so discovery returns instead of polling until the deadline. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Javier de la Torre Co-authored-by: Chris Lu Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Chris Lu --- weed/worker/tasks/iceberg/row_lineage.go | 35 +++++++++-- .../iceberg/row_lineage_first_id_test.go | 63 +++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) create mode 100644 weed/worker/tasks/iceberg/row_lineage_first_id_test.go diff --git a/weed/worker/tasks/iceberg/row_lineage.go b/weed/worker/tasks/iceberg/row_lineage.go index c751f4a98b..f6aa85c2aa 100644 --- a/weed/worker/tasks/iceberg/row_lineage.go +++ b/weed/worker/tasks/iceberg/row_lineage.go @@ -368,12 +368,16 @@ func copyFileRows(ctx context.Context, reader *parquet.Reader, entry iceberg.Man }) } -// mergedFileFirstRowID is the _row_id of the merged output's first row, or nil when it carries none: the value the -// manifest entry's first_row_id must name, since rewritten rows keep their IDs rather than taking fresh ones. +// mergedFileFirstRowID is the _row_id of the merged output's first row, or nil when the file must be committed +// without a first_row_id: a null _row_id would inherit first_row_id+position, an id another row already holds. +// Committed without one, the nulls take fresh ids from the table's next-row-id and materialized values still +// override inheritance. func mergedFileFirstRowID(data []byte) (*int64, error) { - reader := parquet.NewReader(bytes.NewReader(data)) - defer reader.Close() - schema := reader.Schema() + file, err := parquet.OpenFile(bytes.NewReader(data), int64(len(data))) + if err != nil { + return nil, err + } + schema := file.Schema() if schema == nil { return nil, nil } @@ -381,6 +385,11 @@ func mergedFileFirstRowID(data []byte) (*int64, error) { if !ok { return nil, nil } + if rowIDHasNulls(file, leaf.ColumnIndex) { + return nil, nil + } + reader := parquet.NewReader(file) + defer reader.Close() rows := make([]parquet.Row, 1) n, err := reader.ReadRows(rows) if err != nil && err != io.EOF { @@ -402,6 +411,22 @@ func mergedFileFirstRowID(data []byte) (*int64, error) { return nil, nil } +// rowIDHasNulls reports whether any row group's _row_id column chunk counts a null. A chunk whose statistics +// cannot be read counts as having one: a first_row_id named on a hidden null is the failure this guards. +func rowIDHasNulls(file *parquet.File, columnIndex int) bool { + for _, rg := range file.RowGroups() { + chunks := rg.ColumnChunks() + if columnIndex >= len(chunks) { + return true + } + cc, ok := chunks[columnIndex].(*parquet.FileColumnChunk) + if !ok || cc.NullCount() > 0 { + return true + } + } + return false +} + // lineageCodec is the codec of the first data column leaf that declares one, // ZSTD when none does. func lineageCodec(base []parquet.Field) compress.Codec { diff --git a/weed/worker/tasks/iceberg/row_lineage_first_id_test.go b/weed/worker/tasks/iceberg/row_lineage_first_id_test.go new file mode 100644 index 0000000000..07d9929640 --- /dev/null +++ b/weed/worker/tasks/iceberg/row_lineage_first_id_test.go @@ -0,0 +1,63 @@ +package iceberg + +import ( + "bytes" + "testing" + + "github.com/parquet-go/parquet-go" +) + +type lineageRow struct { + ID int64 `parquet:"id"` + RowID *int64 `parquet:"_row_id,optional"` +} + +func lineageFile(t *testing.T, ids ...*int64) []byte { + t.Helper() + var buf bytes.Buffer + w := parquet.NewGenericWriter[lineageRow](&buf) + rows := make([]lineageRow, len(ids)) + for i, id := range ids { + rows[i] = lineageRow{ID: int64(i), RowID: id} + } + if _, err := w.Write(rows); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func i64(v int64) *int64 { return &v } + +// A merged v3 file names its first row's id as first_row_id only when every +// row carries a materialized id: a null row would inherit first_row_id plus +// its position, an id another row already holds (high review of the 4.49 +// series; 4.49's own code took the first row's id whatever followed). +func TestMergedFileFirstRowIDOnlyWithoutNullIDs(t *testing.T) { + cases := []struct { + name string + ids []*int64 + want *int64 + }{ + {"every row materialized", []*int64{i64(40), i64(41), i64(42)}, i64(40)}, + {"materialized first, a null later", []*int64{i64(40), nil, i64(42)}, nil}, + {"null first", []*int64{nil, i64(41)}, nil}, + {"all null", []*int64{nil, nil}, nil}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + got, err := mergedFileFirstRowID(lineageFile(t, c.ids...)) + if err != nil { + t.Fatal(err) + } + switch { + case c.want == nil && got != nil: + t.Fatalf("first_row_id %d, want none", *got) + case c.want != nil && (got == nil || *got != *c.want): + t.Fatalf("first_row_id %v, want %d", got, *c.want) + } + }) + } +} From 7a9a9f10af150e834477028f37dd0eda4c2af06f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 23 Oct 2026 23:05:56 -0700 Subject: [PATCH 47/70] ci: invoke pip via python -m in s3tests clone steps (#12010) * ci: invoke pip via python -m in s3tests clone steps The self-hosted runner reaches the Clone s3-tests step with the setup-python toolcache interpreter on PATH but no pip shim, so the bare 'pip install' exits 127. python -m pip resolves through the interpreter that is already on PATH, matching the helm_ci workflow. * ci: invoke pip via python3 -m in s3tests clone steps The adjacent fix_s3_tests_bucket_conflicts step already runs python3, so python3 is the interpreter guaranteed on the self-hosted PATH; a missing python alias would fail the same way the missing pip shim did. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/s3tests.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/s3tests.yml b/.github/workflows/s3tests.yml index 26f7a83553..971cd033d5 100644 --- a/.github/workflows/s3tests.yml +++ b/.github/workflows/s3tests.yml @@ -58,9 +58,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Fix S3 tests bucket creation conflicts run: | @@ -411,9 +411,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Fix S3 tests bucket creation conflicts run: | @@ -582,9 +582,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Run S3 CORS tests timeout-minutes: 10 @@ -811,9 +811,9 @@ jobs: cd s3-tests sudo apt-get update -qq sudo apt-get install -y -qq libxml2-dev libxslt1-dev zlib1g-dev - pip install -r requirements.txt - pip install tox - pip install -e . + python3 -m pip install -r requirements.txt + python3 -m pip install tox + python3 -m pip install -e . - name: Run Ceph S3 tests with SQL store timeout-minutes: 15 From fb3fb9dcb85fae5af01e7d42abdfc8465669a2c8 Mon Sep 17 00:00:00 2001 From: Junker der Provinz Date: Sat, 24 Oct 2026 11:04:39 +0200 Subject: [PATCH 48/70] s3api: support s3:RequestObjectTagKeys in bucket policies (#11993) * s3api: support s3:RequestObjectTagKeys in bucket policies Condition operators accept the ForAllValues: and ForAnyValue: qualifiers, and s3:RequestObjectTagKeys holds the keys of the tags taken from X-Amz-Tagging wherever s3:RequestObjectTag/ is set. CopyObject no longer stores an empty-key tag for an empty segment in X-Amz-Tagging, so it keeps the key set a policy has checked. Fixes #7679 * s3: parse tagging header like the tag handlers, not url.ParseQuery url.ParseQuery stops at 10,000 parameters, so an X-Amz-Tagging header padded with empty & segments made requestObjectTags return nil: the copy handler still stored the denied tag while every s3:RequestObjectTag* condition saw nothing. Scan non-empty segments the way parseTagsHeader does so the policy sees the same tags the object gets. * test: gofmt s3tables test files after #11997 The format check fails on these files on master as merged, so every PR's S3 Tables Format Check goes red until they are re-formatted. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../policy_engine/README_POLICY_ENGINE.md | 3 +- weed/s3api/policy_engine/conditions.go | 59 ++++++- .../policy_engine/conditions_set_test.go | 102 ++++++++++++ weed/s3api/policy_engine/engine.go | 37 +++-- weed/s3api/policy_engine/engine_test.go | 153 ++++++++++++++++++ weed/s3api/tags.go | 5 + weed/s3api/tags_test.go | 9 ++ 7 files changed, 353 insertions(+), 15 deletions(-) create mode 100644 weed/s3api/policy_engine/conditions_set_test.go diff --git a/weed/s3api/policy_engine/README_POLICY_ENGINE.md b/weed/s3api/policy_engine/README_POLICY_ENGINE.md index 97c4344e4a..78e1df5545 100644 --- a/weed/s3api/policy_engine/README_POLICY_ENGINE.md +++ b/weed/s3api/policy_engine/README_POLICY_ENGINE.md @@ -125,6 +125,7 @@ Support for all AWS condition operators: - **Boolean**: `Bool` - **ARN**: `ArnEquals`, `ArnLike` - **Null**: `Null` +- **Set qualifiers**: `ForAllValues:` and `ForAnyValue:` before any of the above, for multi-valued keys such as `s3:RequestObjectTagKeys` ### 4. Condition Keys @@ -357,6 +358,7 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | `aws:CurrentTime` | Time-based access control | | `s3:ExistingObjectTag/` | Tag-based access control for existing objects | | `s3:RequestObjectTag/` | Tag conditions on the tags a PUT request sets | +| `s3:RequestObjectTagKeys` | The set of tag keys a PUT request sets | | Wildcard Patterns | Support for `*` and `?` in actions and resources | | Principal Matching | `*`, account IDs, and user ARNs | @@ -364,7 +366,6 @@ The policy engine provides a seamless upgrade path from SeaweedFS's existing sim | Feature | GitHub Issue | |---------|--------------| -| `s3:RequestObjectTagKeys` | Check which tag keys are in request | | `s3:x-amz-content-sha256` | Content hash condition | | `s3:x-amz-server-side-encryption` | SSE condition | | `s3:x-amz-storage-class` | Storage class condition | diff --git a/weed/s3api/policy_engine/conditions.go b/weed/s3api/policy_engine/conditions.go index a995c5ddb9..bfb7bf0b94 100644 --- a/weed/s3api/policy_engine/conditions.go +++ b/weed/s3api/policy_engine/conditions.go @@ -669,8 +669,58 @@ func (e *NullEvaluator) Evaluate(conditionValue interface{}, contextValues []str return false } -// GetConditionEvaluator returns the appropriate evaluator for a condition operator +// forAllValuesEvaluator matches when every request value satisfies the base +// operator on its own, and so also when the request carries no value. +type forAllValuesEvaluator struct { + base ConditionEvaluator +} + +func (e *forAllValuesEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool { + for _, v := range contextValues { + if !e.base.Evaluate(conditionValue, []string{v}) { + return false + } + } + return true +} + +// forAnyValueEvaluator matches when at least one request value satisfies the +// base operator on its own. Checking values one at a time keeps a negated +// base such as StringNotEquals from meaning "none of them". +type forAnyValueEvaluator struct { + base ConditionEvaluator +} + +func (e *forAnyValueEvaluator) Evaluate(conditionValue interface{}, contextValues []string) bool { + for _, v := range contextValues { + if e.base.Evaluate(conditionValue, []string{v}) { + return true + } + } + return false +} + +// GetConditionEvaluator returns the appropriate evaluator for a condition +// operator, including the ForAllValues: and ForAnyValue: set qualifiers. func GetConditionEvaluator(operator string) (ConditionEvaluator, error) { + if base, ok := strings.CutPrefix(operator, "ForAllValues:"); ok { + evaluator, err := baseConditionEvaluator(base) + if err != nil { + return nil, err + } + return &forAllValuesEvaluator{base: evaluator}, nil + } + if base, ok := strings.CutPrefix(operator, "ForAnyValue:"); ok { + evaluator, err := baseConditionEvaluator(base) + if err != nil { + return nil, err + } + return &forAnyValueEvaluator{base: evaluator}, nil + } + return baseConditionEvaluator(operator) +} + +func baseConditionEvaluator(operator string) (ConditionEvaluator, error) { switch operator { case "StringEquals": return &StringEqualsEvaluator{}, nil @@ -732,6 +782,10 @@ const ExistingObjectTagPrefix = "s3:ExistingObjectTag/" // request sets through the X-Amz-Tagging header. const RequestObjectTagPrefix = "s3:RequestObjectTag/" +// RequestObjectTagKeys is the multi-valued condition key listing the keys of +// the tags a request sets through the X-Amz-Tagging header. +const RequestObjectTagKeys = "s3:RequestObjectTagKeys" + // getConditionContextValue resolves the value(s) for a condition key. // For s3:ExistingObjectTag/ conditions, it looks up the tag in objectEntry. // For other condition keys, it looks up the value in contextValues. @@ -780,7 +834,8 @@ func evaluateConditions(conditions PolicyConditions, contextValues map[string][] for key, value := range conditionMap { contextVals := getConditionContextValue(key, contextValues, objectEntry) if original := originalGrants[key]; len(original) != 0 && isGrantConditionKey(key) { - switch operator { + // Extra values can only make ForAllValues fail, so it keeps the canonical list. + switch strings.TrimPrefix(operator, "ForAnyValue:") { case "StringEquals", "StringEqualsIgnoreCase", "StringLike", "ArnEquals", "ArnLike": contextVals = append(append([]string(nil), contextVals...), original...) } diff --git a/weed/s3api/policy_engine/conditions_set_test.go b/weed/s3api/policy_engine/conditions_set_test.go new file mode 100644 index 0000000000..80aac29a0f --- /dev/null +++ b/weed/s3api/policy_engine/conditions_set_test.go @@ -0,0 +1,102 @@ +package policy_engine + +import ( + "testing" +) + +func TestConditionSetOperators(t *testing.T) { + allowed := []string{"department", "project"} + tests := []struct { + name string + operator string + conditionValue interface{} + contextValues []string + expected bool + }{ + {"ForAllValues all match", "ForAllValues:StringEquals", allowed, []string{"department", "project"}, true}, + {"ForAllValues subset matches", "ForAllValues:StringEquals", allowed, []string{"project"}, true}, + {"ForAllValues one outsider fails", "ForAllValues:StringEquals", allowed, []string{"department", "owner"}, false}, + {"ForAllValues empty set matches", "ForAllValues:StringEquals", allowed, []string{}, true}, + {"ForAllValues nil set matches", "ForAllValues:StringEquals", allowed, nil, true}, + {"ForAllValues is case sensitive", "ForAllValues:StringEquals", allowed, []string{"Department"}, false}, + {"ForAllValues StringLike all match", "ForAllValues:StringLike", "proj-*", []string{"proj-a", "proj-b"}, true}, + {"ForAllValues StringLike one outsider fails", "ForAllValues:StringLike", "proj-*", []string{"proj-a", "team"}, false}, + {"ForAllValues StringNotEquals none listed", "ForAllValues:StringNotEquals", allowed, []string{"owner", "team"}, true}, + {"ForAllValues StringNotEquals one listed fails", "ForAllValues:StringNotEquals", allowed, []string{"owner", "project"}, false}, + + {"ForAnyValue one match", "ForAnyValue:StringEquals", allowed, []string{"owner", "project"}, true}, + {"ForAnyValue no match", "ForAnyValue:StringEquals", allowed, []string{"owner", "team"}, false}, + {"ForAnyValue empty set fails", "ForAnyValue:StringEquals", allowed, []string{}, false}, + {"ForAnyValue nil set fails", "ForAnyValue:StringEquals", allowed, nil, false}, + {"ForAnyValue StringLike one match", "ForAnyValue:StringLike", []string{"secret*"}, []string{"owner", "secret-level"}, true}, + {"ForAnyValue StringLike no match", "ForAnyValue:StringLike", []string{"secret*"}, []string{"owner"}, false}, + {"ForAnyValue StringNotEquals one unlisted", "ForAnyValue:StringNotEquals", allowed, []string{"department", "owner"}, true}, + {"ForAnyValue StringNotEquals all listed fails", "ForAnyValue:StringNotEquals", allowed, []string{"department", "project"}, false}, + {"ForAnyValue StringEqualsIgnoreCase", "ForAnyValue:StringEqualsIgnoreCase", allowed, []string{"PROJECT"}, true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + evaluator, err := GetConditionEvaluator(tt.operator) + if err != nil { + t.Fatalf("Failed to get condition evaluator: %v", err) + } + if got := evaluator.Evaluate(tt.conditionValue, tt.contextValues); got != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, got) + } + }) + } +} + +func TestGetConditionEvaluatorRejectsUnknownSetOperators(t *testing.T) { + for _, op := range []string{ + "ForAllValues:StringBogus", + "ForAnyValue:", + "ForEachValue:StringEquals", + "ForAllValues:ForAnyValue:StringEquals", + } { + if _, err := GetConditionEvaluator(op); err == nil { + t.Errorf("GetConditionEvaluator accepted %q; expected error", op) + } + } +} + +func TestValidateBucketPolicyAcceptsSetOperators(t *testing.T) { + policy := `{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringNotEquals": {"s3:RequestObjectTagKeys": ["department", "project"]}, + "ForAnyValue:StringLike": {"s3:RequestObjectTagKeys": ["secret*"]} + } + }] + }` + engine := NewPolicyEngine() + if err := engine.SetBucketPolicy("test-bucket", policy); err != nil { + t.Fatalf("SetBucketPolicy rejected set operators: %v", err) + } +} + +func TestOriginalGrantsSupplementOnlyForAnyValue(t *testing.T) { + const key = "s3:x-amz-grant-read" + contextValues := map[string][]string{key: {"id=canonical"}} + originalGrants := map[string][]string{key: {"uri=original"}} + + anyValue := PolicyConditions{ + "ForAnyValue:StringEquals": {key: NewStringOrStringSlice("uri=original")}, + } + if !evaluateConditions(anyValue, contextValues, nil, nil, originalGrants) { + t.Error("ForAnyValue should see the original grant values") + } + + allValues := PolicyConditions{ + "ForAllValues:StringEquals": {key: NewStringOrStringSlice("id=canonical")}, + } + if !evaluateConditions(allValues, contextValues, nil, nil, originalGrants) { + t.Error("ForAllValues should be evaluated on the canonical values only") + } +} diff --git a/weed/s3api/policy_engine/engine.go b/weed/s3api/policy_engine/engine.go index 0275f33627..e544b227bd 100644 --- a/weed/s3api/policy_engine/engine.go +++ b/weed/s3api/policy_engine/engine.go @@ -2,9 +2,11 @@ package policy_engine import ( "fmt" + "maps" "net/http" "net/url" "regexp" + "slices" "strings" "sync" "sync/atomic" @@ -441,8 +443,11 @@ func (engine *PolicyEngine) ExtractConditionValuesFromRequest(r *http.Request) m // Note: s3:ExistingObjectTag/ conditions are evaluated using objectEntry // passed to EvaluatePolicy, not extracted from the request. - for key, value := range requestObjectTags(r) { - values[RequestObjectTagPrefix+key] = []string{value} + if tags := requestObjectTags(r); len(tags) > 0 { + for key, value := range tags { + values[RequestObjectTagPrefix+key] = []string{value} + } + values[RequestObjectTagKeys] = slices.Sorted(maps.Keys(tags)) } // S3 bucket-level conditions @@ -514,16 +519,24 @@ func requestObjectTags(r *http.Request) map[string]string { if !storesRequestObjectTags(r) { return nil } - // A malformed header is rejected by the handler with InvalidTag. - parsed, err := url.ParseQuery(tagging) - if err != nil { - return nil - } - // putToFiler rejects a repeated key, but CopyObject stores its last - // value, so that is the one the policy has to see. - tags := make(map[string]string, len(parsed)) - for key, vals := range parsed { - tags[key] = vals[len(vals)-1] + // Parse the same non-empty segments the tag handlers store: url.ParseQuery + // caps at 10,000 parameters, so a header padded with "&" segments would + // hide stored tag keys from every s3:RequestObjectTag* condition. A + // malformed pair is rejected by the handler with InvalidTag. + tags := make(map[string]string) + for _, segment := range strings.Split(tagging, "&") { + if segment == "" { + continue + } + rawKey, rawValue, _ := strings.Cut(segment, "=") + key, errKey := url.QueryUnescape(rawKey) + value, errValue := url.QueryUnescape(rawValue) + if errKey != nil || errValue != nil { + return nil + } + // putToFiler rejects a repeated key, but CopyObject stores its last + // value, so that is the one the policy has to see. + tags[key] = value } return tags } diff --git a/weed/s3api/policy_engine/engine_test.go b/weed/s3api/policy_engine/engine_test.go index 587ffdc115..6f4cccc687 100644 --- a/weed/s3api/policy_engine/engine_test.go +++ b/weed/s3api/policy_engine/engine_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "strings" "testing" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" @@ -1051,6 +1052,158 @@ func TestRequestObjectTagConditionDeny(t *testing.T) { } } +func evaluateTaggedRequest(engine *PolicyEngine, method, target string, headers map[string]string, tagging string) PolicyEvaluationResult { + if method == "" { + method = http.MethodPut + } + req := httptest.NewRequest(method, "/test-bucket/test-object"+target, nil) + for k, v := range headers { + req.Header.Set(k, v) + } + if tagging != "" { + req.Header.Set(s3_constants.AmzObjectTagging, tagging) + } + return engine.EvaluatePolicy("test-bucket", &PolicyEvaluationArgs{ + Action: "s3:PutObject", + Resource: "arn:aws:s3:::test-bucket/test-object", + Principal: "*", + Conditions: engine.ExtractConditionValuesFromRequest(req), + }) +} + +func TestRequestObjectTagKeysCondition(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringEquals": { + "s3:RequestObjectTagKeys": ["department", "project"] + }, + "Null": { + "s3:RequestObjectTagKeys": "false" + } + } + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAnyValue:StringLike": { + "s3:RequestObjectTagKeys": ["secret*"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + copyHeaders := map[string]string{"X-Amz-Copy-Source": "/src-bucket/src-object"} + replaceHeaders := map[string]string{ + "X-Amz-Copy-Source": "/src-bucket/src-object", + s3_constants.AmzObjectTaggingDirective: "REPLACE", + } + + tests := []struct { + name string + method string + target string + headers map[string]string + tagging string + expected PolicyEvaluationResult + }{ + {"listed key allows", "", "", nil, "department=engineering", PolicyResultAllow}, + {"all listed keys allow", "", "", nil, "department=engineering&project=alpha", PolicyResultAllow}, + {"unlisted key is not allowed", "", "", nil, "department=engineering&owner=bob", PolicyResultIndeterminate}, + {"no tagging header is not allowed", "", "", nil, "", PolicyResultIndeterminate}, + {"repeated key counts once", "", "", nil, "department=a&department=b", PolicyResultAllow}, + {"url-encoded key is decoded", "", "", nil, "depart%6Dent=engineering", PolicyResultAllow}, + {"empty segments add no key", "", "", nil, "&department=engineering&", PolicyResultAllow}, + {"empty key is not allowed", "", "", nil, "department=engineering&=x", PolicyResultIndeterminate}, + {"key case is preserved", "", "", nil, "Department=engineering", PolicyResultIndeterminate}, + {"malformed header is not allowed", "", "", nil, "department=engineering&%zz", PolicyResultIndeterminate}, + {"secret key is denied", "", "", nil, "secret=1", PolicyResultDeny}, + {"secret key behind a padded header is denied", "", "", nil, "secret=1" + strings.Repeat("&", 10001), PolicyResultDeny}, + {"secret key among listed keys is denied", "", "", nil, "department=engineering&secret-level=2", PolicyResultDeny}, + {"url-encoded secret key is denied", "", "", nil, "secr%65t=1", PolicyResultDeny}, + {"secret key on a copy that replaces tags is denied", "", "", replaceHeaders, "secret=1", PolicyResultDeny}, + {"secret key on a multipart create is denied", "POST", "?uploads", nil, "secret=1", PolicyResultDeny}, + {"header is ignored on a copy that keeps source tags", "", "", copyHeaders, "secret=1", PolicyResultIndeterminate}, + {"header is ignored on PutObjectTagging", "", "?tagging", nil, "department=engineering", PolicyResultIndeterminate}, + {"header is ignored on a part upload", "", "?partNumber=1&uploadId=x", nil, "secret=1", PolicyResultIndeterminate}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := evaluateTaggedRequest(engine, tt.method, tt.target, tt.headers, tt.tagging); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + +func TestRequestObjectTagKeysRequiredTagDeny(t *testing.T) { + engine := NewPolicyEngine() + + policyJSON := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*" + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:PutObject", + "Resource": "arn:aws:s3:::test-bucket/*", + "Condition": { + "ForAllValues:StringNotEquals": { + "s3:RequestObjectTagKeys": ["department", "project"] + } + } + } + ] + }` + + if err := engine.SetBucketPolicy("test-bucket", policyJSON); err != nil { + t.Fatalf("Failed to set bucket policy: %v", err) + } + + tests := []struct { + name string + tagging string + expected PolicyEvaluationResult + }{ + {"required key passes", "department=engineering", PolicyResultAllow}, + {"required key with another key passes", "project=alpha&owner=bob", PolicyResultAllow}, + {"only other keys are denied", "owner=bob", PolicyResultDeny}, + {"no tagging header is denied", "", PolicyResultDeny}, + {"malformed header is denied", "department=engineering&%zz", PolicyResultDeny}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if result := evaluateTaggedRequest(engine, "", "", nil, tt.tagging); result != tt.expected { + t.Errorf("Expected %v, got %v", tt.expected, result) + } + }) + } +} + // TestExistingObjectTagConditionMultipleTags tests policies with multiple tag conditions func TestExistingObjectTagConditionMultipleTags(t *testing.T) { engine := NewPolicyEngine() diff --git a/weed/s3api/tags.go b/weed/s3api/tags.go index 7b1654529d..76830ccf14 100644 --- a/weed/s3api/tags.go +++ b/weed/s3api/tags.go @@ -53,6 +53,11 @@ func FromTags(tags map[string]string) (t *Tagging) { func parseTagsHeader(tags string) (map[string]string, error) { parsedTags := make(map[string]string) for _, v := range util.StringSplit(tags, "&") { + // url.ParseQuery skips empty segments too, so a copy stores the + // same tag keys that PutObject and bucket policies see. + if v == "" { + continue + } key, value, hasValue := strings.Cut(v, "=") decodedKey, err := url.QueryUnescape(key) if err != nil { diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index 2002d65f28..a7dd1b806e 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -71,6 +71,15 @@ func TestParseTagsHeader(t *testing.T) { }, expectError: false, }, + { + name: "empty segments add no tag", + input: "&key1=value1&&key2=value2&", + expected: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + expectError: false, + }, { name: "unescaped equals in value", input: "token=part1=part2&normal=test", From be081b49b752483b1837c0789e4b25be9a4bbb30 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 02:12:17 -0700 Subject: [PATCH 49/70] notification: named webhook targets for per-path event routing (#11925) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * notification: honor the config prefix in the multi-queue guard validateOneEnabledQueue checked ".enabled" without the notification prefix, so the fatal-on-two-queues guard never saw the real keys and could never fire. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: support named webhook targets under [notification.webhook.] The webhook queue only accepted one endpoint, so events could not be routed to different services by path prefix. Each enabled [notification.webhook.] subsection now spins up its own target — own filter, http client, retry middleware and worker budget — published on a per-target topic so every matching endpoint gets each event exactly once. The flat [notification.webhook] section keeps working as the unnamed target, and named subsections alone now activate the queue. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: require enabled for the flat webhook target buildTargets included the flat section whenever an endpoint was set, so a disabled [notification.webhook] block — which the scaffold ships with a placeholder endpoint — would start receiving events as soon as any named target activated the queue. * notification: resolve named target enabled flags through config namedTargetPrefixes read the raw map value, which never sees environment variable overrides; GetBool resolves WEED_* env bindings, so a named target disabled in the file can now be switched on by WEED_NOTIFICATION_WEBHOOK__ENABLED. * notification: cancel the webhook queue only after all workers stop Each target's router goroutine deferred the shared cancel, so the first worker to exit tore down the queue context for every other target and the dead letter logger. * notification: isolate webhook targets behind per-target queues SendMessage published to each target's topic inline, so a subscriber channel that filled up blocked every later target and the filer's notification path with it. Each target now drains its own inbox through a forwarder goroutine; SendMessage fails fast on a full inbox, matching the "throw message" contract the filer already applies to errors. * notification: name the webhook target in failure and dead letter logs Every target dead-letters to the same topic, so without the name there is no way to tell which endpoint failed or needs a replay. * notification: assert delivery in the named targets test The test only exercised SendMessage; it now records requests on both endpoints and checks matching, non-matching, and unrouted keys reach the right servers exactly once. * notification: wait for the webhook router before accepting events * notification: apply backpressure instead of dropping a full webhook inbox * notification: bound queued and in-flight webhook deliveries per target * notification: release the pending token when shutdown races the enqueue A SendMessage that wins the pending select as the queue is canceling could block forever on a full, undrained inbox. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: stop waiting for a router that failed to start A router that exits before reaching Running() would leave initialize blocked on a context nothing had canceled yet. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * notification: route webhook events past saturated targets A full pending reservation made SendMessage wait on the slow target before it ever reached the healthy ones after it, coupling every endpoint's latency to the most congested one. Targets that can take a reservation now get the event immediately; saturated targets defer to a second pass that still blocks for a slot, so backpressure is kept and no event is dropped. * notification: retry webhook deliveries before dead-lettering PoisonQueue must wrap the retry middleware: added after it, the poison middleware intercepts the first handler error, republishes to the dead letter topic, and returns nil — the retry chain never sees a failure, so max_retries/backoff settings were dead code and every delivery failed straight to dead letter on the first attempt. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/scaffold/notification.toml | 13 + weed/notification/configuration.go | 24 +- weed/notification/webhook/types.go | 7 - weed/notification/webhook/webhook_queue.go | 371 +++++++++++---- .../webhook/webhook_queue_test.go | 439 ++++++++++++++---- 5 files changed, 657 insertions(+), 197 deletions(-) diff --git a/weed/command/scaffold/notification.toml b/weed/command/scaffold/notification.toml index ebbd240f9f..7d327984e4 100644 --- a/weed/command/scaffold/notification.toml +++ b/weed/command/scaffold/notification.toml @@ -81,3 +81,16 @@ workers = 5 # optional: concurrent worker buffer_size = 10000 # optional: event buffer size (default: 10000, range: 100-1000000) # event_types = ["create", "update", "delete", "rename"] # optional: filter by event types (default: all) # path_prefixes = ["/important", "/data"] # optional: filter by path prefixes (default: all) +# +# Additional endpoints can be declared as named subsections, each with its own +# endpoint, filters, retry and worker settings, e.g.: +# [notification.webhook.file_ingress] +# enabled = true +# endpoint = "https://ingress.example.com/hook" +# event_types = ["create"] +# path_prefixes = ["/buckets/uploads/data"] +# +# [notification.webhook.metadata_extractor] +# enabled = true +# endpoint = "https://metadata.example.com/hook" +# path_prefixes = ["/buckets/repository"] diff --git a/weed/notification/configuration.go b/weed/notification/configuration.go index 1c620f2e6a..d271323149 100644 --- a/weed/notification/configuration.go +++ b/weed/notification/configuration.go @@ -26,10 +26,10 @@ func LoadConfiguration(config *util.ViperProxy, prefix string) { return } - validateOneEnabledQueue(config) + validateOneEnabledQueue(config, prefix) for _, queue := range MessageQueues { - if config.GetBool(prefix + queue.GetName() + ".enabled") { + if queueEnabled(config, prefix, queue) { if err := queue.Initialize(config, prefix+queue.GetName()+"."); err != nil { glog.Fatalf("Failed to initialize notification for %s: %+v", queue.GetName(), err) @@ -42,10 +42,26 @@ func LoadConfiguration(config *util.ViperProxy, prefix string) { } -func validateOneEnabledQueue(config *util.ViperProxy) { +// queueEnabled reports whether a queue is switched on — by the flat +// ".enabled" flag or, for queues accepting named sub-instances +// ([notification..]), by any enabled subconfig. +func queueEnabled(config util.Configuration, prefix string, queue MessageQueue) bool { + key := prefix + queue.GetName() + "." + if config.GetBool(key + "enabled") { + return true + } + if q, ok := queue.(interface { + HasEnabledSubconfig(util.Configuration, string) bool + }); ok { + return q.HasEnabledSubconfig(config, key) + } + return false +} + +func validateOneEnabledQueue(config *util.ViperProxy, prefix string) { enabledQueue := "" for _, queue := range MessageQueues { - if config.GetBool(queue.GetName() + ".enabled") { + if queueEnabled(config, prefix, queue) { if enabledQueue == "" { enabledQueue = queue.GetName() } else { diff --git a/weed/notification/webhook/types.go b/weed/notification/webhook/types.go index f177f635a4..1567e832f2 100644 --- a/weed/notification/webhook/types.go +++ b/weed/notification/webhook/types.go @@ -3,7 +3,6 @@ package webhook import ( "fmt" "net/url" - "strconv" "github.com/seaweedfs/seaweedfs/weed/notification" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -30,12 +29,6 @@ func (e eventType) valid() bool { return notification.ValidEventType(string(e)) } -var ( - pubSubHandlerNameTemplate = func(n int) string { - return "webhook_handler_" + strconv.Itoa(n) - } -) - type client interface { sendMessage(message *webhookMessage) error } diff --git a/weed/notification/webhook/webhook_queue.go b/weed/notification/webhook/webhook_queue.go index 2ac9db3ebb..afaa326245 100644 --- a/weed/notification/webhook/webhook_queue.go +++ b/weed/notification/webhook/webhook_queue.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + "sort" + "strings" + "sync" "time" "github.com/ThreeDotsLabs/watermill" @@ -22,45 +25,176 @@ func init() { notification.MessageQueues = append(notification.MessageQueues, &Queue{}) } +// target is one webhook endpoint with its own filter, http client and +// concurrency budget. The flat [notification.webhook] section is the unnamed +// target; each enabled [notification.webhook.] subsection adds another. +type target struct { + name string + config *config + filter *filter + client client + sem chan struct{} + inbox chan *message.Message + pending chan struct{} +} + +func newTarget(name string, cfg *config) (*target, error) { + t := &target{ + name: name, + config: cfg, + filter: newFilter(cfg), + sem: make(chan struct{}, cfg.nWorkers), + inbox: make(chan *message.Message, cfg.bufferSize), + pending: make(chan struct{}, cfg.bufferSize), + } + hClient, err := newHTTPClient(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create webhook http client: %w", err) + } + t.client = hClient + return t, nil +} + +func (t *target) topic() string { + if t.name == "" { + return pubSubTopicName + } + return pubSubTopicName + "_" + t.name +} + +func (t *target) handlerName() string { + if t.name == "" { + return "webhook_handler" + } + return "webhook_handler_" + t.name +} + type Queue struct { - router *message.Router + routers []*message.Router queueChannel *gochannel.GoChannel - config *config - client client - filter *filter + targets []*target ctx context.Context cancel context.CancelFunc - - // Semaphore for controlling concurrent webhook requests - sem chan struct{} } func (w *Queue) GetName() string { return queueName } +// HasEnabledSubconfig reports whether any [notification.webhook.] +// subsection is enabled, so the queue activates on named targets even when the +// flat [notification.webhook] section is absent. +func (w *Queue) HasEnabledSubconfig(configuration util.Configuration, prefix string) bool { + return len(namedTargetPrefixes(configuration, prefix)) > 0 +} + +// stringMapConfig is implemented by *util.ViperProxy; anything narrower simply +// gets no named-target discovery. +type stringMapConfig interface { + GetStringMap(key string) map[string]interface{} +} + +func namedTargetPrefixes(configuration util.Configuration, prefix string) []string { + mg, ok := configuration.(stringMapConfig) + if !ok { + return nil + } + var prefixes []string + for k, v := range mg.GetStringMap(strings.TrimSuffix(prefix, ".")) { + if _, ok := v.(map[string]interface{}); !ok { + continue + } + p := prefix + k + "." + if !configuration.GetBool(p + "enabled") { + continue + } + prefixes = append(prefixes, p) + } + sort.Strings(prefixes) + return prefixes +} + +func buildTargets(configuration util.Configuration, prefix string) ([]*target, error) { + var prefixes []string + if configuration.GetBool(prefix + "enabled") { + prefixes = append(prefixes, prefix) + } + prefixes = append(prefixes, namedTargetPrefixes(configuration, prefix)...) + + var targets []*target + for _, p := range prefixes { + cfg := newConfigWithDefaults(configuration, p) + if err := cfg.validate(); err != nil { + return nil, err + } + t, err := newTarget(strings.TrimSuffix(strings.TrimPrefix(p, prefix), "."), cfg) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + return targets, nil +} + func (w *Queue) SendMessage(key string, msg proto.Message) error { eventNotification, ok := msg.(*filer_pb.EventNotification) if !ok { return nil } - if w.filter != nil && !w.filter.shouldPublish(key, eventNotification) { + var deferred []routedMessage + for _, t := range w.targets { + if t.filter != nil && !t.filter.shouldPublish(key, eventNotification) { + continue + } + m := newWebhookMessage(key, msg) + if m == nil { + continue + } + wMsg, err := m.toWaterMillMessage() + if err != nil { + return err + } + wMsg.Metadata.Set("target", t.name) + // a free reservation routes the event without waiting on slower + // targets; a saturated target defers to the blocking pass below so + // it applies backpressure instead of dropping or starving the rest + select { + case t.pending <- struct{}{}: + if err := t.enqueue(w.ctx, wMsg); err != nil { + return err + } + default: + deferred = append(deferred, routedMessage{t, wMsg}) + } + } + for _, r := range deferred { + select { + case r.target.pending <- struct{}{}: + case <-w.ctx.Done(): + return w.ctx.Err() + } + if err := r.target.enqueue(w.ctx, r.msg); err != nil { + return err + } + } + return nil +} + +type routedMessage struct { + target *target + msg *message.Message +} + +func (t *target) enqueue(ctx context.Context, wMsg *message.Message) error { + select { + case t.inbox <- wMsg: return nil + case <-ctx.Done(): + t.releasePending() + return ctx.Err() } - - m := newWebhookMessage(key, msg) - if m == nil { - return nil - } - - wMsg, err := m.toWaterMillMessage() - if err != nil { - return err - } - - return w.queueChannel.Publish(pubSubTopicName, wMsg) } func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) { @@ -78,105 +212,158 @@ func (w *webhookMessage) toWaterMillMessage() (*message.Message, error) { } func (w *Queue) Initialize(configuration util.Configuration, prefix string) error { - c := newConfigWithDefaults(configuration, prefix) - - if err := c.validate(); err != nil { + targets, err := buildTargets(configuration, prefix) + if err != nil { return err } + if len(targets) == 0 { + return fmt.Errorf("webhook endpoint is required") + } - return w.initialize(c) + return w.initialize(targets) } -func (w *Queue) initialize(cfg *config) error { +func (w *Queue) initialize(targets []*target) error { w.ctx, w.cancel = context.WithCancel(context.Background()) - w.config = cfg - w.filter = newFilter(cfg) + w.targets = targets - // Initialize semaphore for controlling concurrent webhook requests - w.sem = make(chan struct{}, cfg.nWorkers) - - hClient, err := newHTTPClient(cfg) - if err != nil { - return fmt.Errorf("failed to create webhook http client: %w", err) - } - w.client = hClient - - if err = w.setupWatermillQueue(cfg); err != nil { + if err := w.setupWatermillQueue(targets); err != nil { return fmt.Errorf("failed to setup watermill queue: %w", err) } - if err = w.logDeadLetterMessages(); err != nil { + if err := w.logDeadLetterMessages(); err != nil { return err } return nil } -func (w *Queue) setupWatermillQueue(cfg *config) error { +func (w *Queue) setupWatermillQueue(targets []*target) error { logger := watermill.NewStdLogger(false, false) + bufferSize := 0 + for _, t := range targets { + if t.config.bufferSize > bufferSize { + bufferSize = t.config.bufferSize + } + } pubSubConfig := gochannel.Config{ - OutputChannelBuffer: int64(cfg.bufferSize), + OutputChannelBuffer: int64(bufferSize), Persistent: false, } w.queueChannel = gochannel.NewGoChannel(pubSubConfig, logger) - router, err := message.NewRouter( - message.RouterConfig{ - CloseTimeout: 60 * time.Second, - }, - logger, - ) - if err != nil { - return fmt.Errorf("failed to create router: %w", err) - } - w.router = router + var wg sync.WaitGroup + for _, t := range targets { + // One router per target so each keeps its own retry/backoff settings. + // Handlers subscribe to the target's own topic, so a message is + // delivered to an endpoint exactly once per matching target. + router, err := message.NewRouter( + message.RouterConfig{ + CloseTimeout: 60 * time.Second, + }, + logger, + ) + if err != nil { + return fmt.Errorf("failed to create router: %w", err) + } + w.routers = append(w.routers, router) - retryMiddleware := middleware.Retry{ - MaxRetries: cfg.maxRetries, - InitialInterval: time.Duration(cfg.backoffSeconds) * time.Second, - MaxInterval: time.Duration(cfg.maxBackoffSeconds) * time.Second, - Multiplier: 2.0, - RandomizationFactor: 0.3, - Logger: logger, - }.Middleware + retryMiddleware := middleware.Retry{ + MaxRetries: t.config.maxRetries, + InitialInterval: time.Duration(t.config.backoffSeconds) * time.Second, + MaxInterval: time.Duration(t.config.maxBackoffSeconds) * time.Second, + Multiplier: 2.0, + RandomizationFactor: 0.3, + Logger: logger, + }.Middleware - poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic) - if err != nil { - return fmt.Errorf("failed to create poison queue: %w", err) - } - - router.AddPlugin(plugin.SignalsHandler) - router.AddMiddleware(retryMiddleware, poisonQueue) - - // Add a single handler to avoid duplicate message delivery. - // With gochannel's default behavior, each handler call creates - // a separate subscription, and all subscriptions receive their own copy of each message. - // Using a single handler ensures each webhook is sent only once. - // Concurrency is controlled via semaphore in handleWebhook based on nWorkers config. - router.AddConsumerHandler( - "webhook_handler", - pubSubTopicName, - w.queueChannel, - w.handleWebhook, - ) - - go func() { - // cancels the queue context so the dead letter logger exists in case context not canceled by the shutdown signal already - defer w.cancel() - - if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) { - glog.Errorf("webhook pubsub worker stopped with error: %v", err) + poisonQueue, err := middleware.PoisonQueue(w.queueChannel, deadLetterTopic) + if err != nil { + return fmt.Errorf("failed to create poison queue: %w", err) } - glog.Info("webhook pubsub worker stopped") + router.AddPlugin(plugin.SignalsHandler) + // outermost: the reservation SendMessage took releases once the whole + // chain (retries, dead letter) is done, bounding in-flight deliveries. + // poison wraps retry so the delivery is retried before dead-lettering. + releasePending := func(h message.HandlerFunc) message.HandlerFunc { + return func(msg *message.Message) ([]*message.Message, error) { + defer t.releasePending() + return h(msg) + } + } + router.AddMiddleware(releasePending, poisonQueue, retryMiddleware) + + t := t + router.AddConsumerHandler( + t.handlerName(), + t.topic(), + w.queueChannel, + t.handleWebhook, + ) + + routerStopped := make(chan struct{}) + wg.Add(1) + go func() { + defer wg.Done() + defer close(routerStopped) + if err := router.Run(w.ctx); err != nil && !errors.Is(err, context.Canceled) { + glog.Errorf("webhook pubsub worker stopped with error: %v", err) + } + glog.Info("webhook pubsub worker stopped") + }() + + go w.forwardTargetMessages(t) + + // without a subscriber GoChannel drops the publish as undeliverable + select { + case <-router.Running(): + case <-routerStopped: + return fmt.Errorf("webhook router for target %q stopped before it started", t.name) + case <-w.ctx.Done(): + return w.ctx.Err() + } + } + + // the queue context lives until every target's worker has stopped so the + // dead letter logger keeps running while any of them is still up + go func() { + wg.Wait() + w.cancel() }() return nil } -func (w *Queue) handleWebhook(msg *message.Message) error { +// forwardTargetMessages publishes each queued event to the target's topic in +// order. Publish detaches its delivery, so the pending reservation is what +// keeps a saturated target from queuing without bound. +func (w *Queue) forwardTargetMessages(t *target) { + for { + select { + case msg := <-t.inbox: + if err := w.queueChannel.Publish(t.topic(), msg); err != nil { + glog.Errorf("webhook target %q failed to publish message: %v", t.name, err) + t.releasePending() + } + case <-w.ctx.Done(): + return + } + } +} + +// releasePending frees one delivery reservation. It never blocks: a nacked +// message GoChannel resends reaches the release again after its token is gone. +func (t *target) releasePending() { + select { + case <-t.pending: + default: + } +} + +func (t *target) handleWebhook(msg *message.Message) error { // Acquire semaphore slot (blocks if at capacity) - w.sem <- struct{}{} - defer func() { <-w.sem }() + t.sem <- struct{}{} + defer func() { <-t.sem }() var n filer_pb.EventNotification if err := proto.Unmarshal(msg.Payload, &n); err != nil { @@ -191,8 +378,8 @@ func (w *Queue) handleWebhook(msg *message.Message) error { Notification: &n, } - if err := w.client.sendMessage(webhookMsg); err != nil { - glog.Errorf("failed to send message to webhook %s: %v", webhookMsg.Key, err) + if err := t.client.sendMessage(webhookMsg); err != nil { + glog.Errorf("webhook target %q failed to send message for %s: %v", t.name, webhookMsg.Key, err) return err } @@ -218,10 +405,12 @@ func (w *Queue) logDeadLetterMessages() error { continue } key := "unknown" + targetName := "" if msg.Metadata != nil { if keyValue, exists := msg.Metadata["key"]; exists { key = keyValue } + targetName = msg.Metadata["target"] } payload := "" if msg.Payload != nil { @@ -232,7 +421,7 @@ func (w *Queue) logDeadLetterMessages() error { payload = n.String() } } - glog.Errorf("received dead letter message: %s, key: %s", payload, key) + glog.Errorf("received dead letter message: %s, key: %s, target: %s", payload, key, targetName) case <-w.ctx.Done(): return } diff --git a/weed/notification/webhook/webhook_queue_test.go b/weed/notification/webhook/webhook_queue_test.go index a084ef975b..ba75ecb147 100644 --- a/weed/notification/webhook/webhook_queue_test.go +++ b/weed/notification/webhook/webhook_queue_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" "time" @@ -170,6 +171,29 @@ func TestWebhookMessageSerialization(t *testing.T) { } } +func newQueueForTest(t *testing.T, cfg *config) *Queue { + t.Helper() + tgt, err := newTarget("", cfg) + if err != nil { + t.Fatalf("newTarget() error = %v", err) + } + q := &Queue{} + if err := q.initialize([]*target{tgt}); err != nil { + t.Fatalf("initialize() error = %v", err) + } + return q +} + +func closeQueueForTest(q *Queue) { + if q.cancel != nil { + q.cancel() + } + time.Sleep(100 * time.Millisecond) + for _, r := range q.routers { + r.Close() + } +} + func TestQueueInitialize(t *testing.T) { cfg := &config{ endpoint: "https://example.com/webhook", @@ -182,32 +206,19 @@ func TestQueueInitialize(t *testing.T) { bufferSize: 100, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Errorf("Initialize() error = %v", err) - } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() - - if q.router == nil { + if len(q.routers) != 1 { t.Error("Expected router to be initialized") } if q.queueChannel == nil { t.Error("Expected queueChannel to be initialized") } - if q.client == nil { + if q.targets[0].client == nil { t.Error("Expected client to be initialized") } - if q.config == nil { + if q.targets[0].config == nil { t.Error("Expected config to be initialized") } } @@ -225,21 +236,8 @@ func TestQueueSendMessage(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) msg := &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{ @@ -247,8 +245,7 @@ func TestQueueSendMessage(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -271,7 +268,7 @@ func TestQueueHandleWebhook(t *testing.T) { } client, _ := newHTTPClient(cfg) - q := &Queue{ + tgt := &target{ client: client, sem: make(chan struct{}, cfg.nWorkers), } @@ -287,8 +284,7 @@ func TestQueueHandleWebhook(t *testing.T) { t.Fatalf("Failed to create watermill message: %v", err) } - err = q.handleWebhook(wmMsg) - if err != nil { + if err := tgt.handleWebhook(wmMsg); err != nil { t.Errorf("handleWebhook() error = %v", err) } } @@ -307,21 +303,8 @@ func TestQueueEndToEnd(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) msg := &filer_pb.EventNotification{ NewEntry: &filer_pb.Entry{ @@ -329,8 +312,7 @@ func TestQueueEndToEnd(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -347,33 +329,20 @@ func TestQueueRetryMechanism(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) // Verify that the queue is properly configured for retries - if q.config.maxRetries != 3 { - t.Errorf("Expected maxRetries=3, got %d", q.config.maxRetries) + if q.targets[0].config.maxRetries != 3 { + t.Errorf("Expected maxRetries=3, got %d", q.targets[0].config.maxRetries) } - if q.config.backoffSeconds != 2 { - t.Errorf("Expected backoffSeconds=2, got %d", q.config.backoffSeconds) + if q.targets[0].config.backoffSeconds != 2 { + t.Errorf("Expected backoffSeconds=2, got %d", q.targets[0].config.backoffSeconds) } - if q.config.maxBackoffSeconds != 10 { - t.Errorf("Expected maxBackoffSeconds=10, got %d", q.config.maxBackoffSeconds) + if q.targets[0].config.maxBackoffSeconds != 10 { + t.Errorf("Expected maxBackoffSeconds=10, got %d", q.targets[0].config.maxBackoffSeconds) } // Test that we can send a message (retry behavior is handled by Watermill middleware) @@ -381,8 +350,7 @@ func TestQueueRetryMechanism(t *testing.T) { NewEntry: &filer_pb.Entry{Name: "test.txt"}, } - err = q.SendMessage("/test/retry", msg) - if err != nil { + if err := q.SendMessage("/test/retry", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } } @@ -419,21 +387,8 @@ func TestQueueNoDuplicateWebhooks(t *testing.T) { bufferSize: 10, } - q := &Queue{} - err := q.initialize(cfg) - if err != nil { - t.Fatalf("Failed to initialize queue: %v", err) - } - - defer func() { - if q.cancel != nil { - q.cancel() - } - time.Sleep(100 * time.Millisecond) - if q.router != nil { - q.router.Close() - } - }() + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) // Wait for router and subscriber to be fully ready time.Sleep(200 * time.Millisecond) @@ -444,8 +399,7 @@ func TestQueueNoDuplicateWebhooks(t *testing.T) { }, } - err = q.SendMessage("/test/path", msg) - if err != nil { + if err := q.SendMessage("/test/path", msg); err != nil { t.Errorf("SendMessage() error = %v", err) } @@ -607,3 +561,298 @@ func TestQueueSendMessageWithFilter(t *testing.T) { }) } } + +type mapConfig struct { + values map[string]interface{} +} + +func (m *mapConfig) GetString(key string) string { + v, _ := m.values[key].(string) + return v +} + +func (m *mapConfig) GetBool(key string) bool { + v, _ := m.values[key].(bool) + return v +} + +func (m *mapConfig) GetInt(key string) int { + v, _ := m.values[key].(int) + return v +} + +func (m *mapConfig) GetStringSlice(key string) []string { + v, _ := m.values[key].([]string) + return v +} + +func (m *mapConfig) SetDefault(key string, value interface{}) {} + +func (m *mapConfig) GetStringMap(key string) map[string]interface{} { + v, _ := m.values[key].(map[string]interface{}) + return v +} + +func TestNamedTargets(t *testing.T) { + var hitsA, hitsB atomic.Int32 + serverA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsA.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverA.Close() + serverB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsB.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverB.Close() + + cfg := &mapConfig{values: map[string]interface{}{ + "notification.webhook": map[string]interface{}{ + "file_ingress": map[string]interface{}{ + "enabled": true, + "endpoint": serverA.URL, + "path_prefixes": []string{"/data/"}, + }, + "metadata_extractor": map[string]interface{}{ + "enabled": true, + "endpoint": serverB.URL, + "path_prefixes": []string{"/repository/"}, + }, + "disabled_one": map[string]interface{}{ + "enabled": false, + "endpoint": "https://example.com/never", + }, + }, + "notification.webhook.file_ingress.enabled": true, + "notification.webhook.file_ingress.endpoint": serverA.URL, + "notification.webhook.file_ingress.path_prefixes": []string{"/data/"}, + "notification.webhook.metadata_extractor.enabled": true, + "notification.webhook.metadata_extractor.endpoint": serverB.URL, + "notification.webhook.metadata_extractor.path_prefixes": []string{"/repository/"}, + }} + + q := &Queue{} + if !q.HasEnabledSubconfig(cfg, "notification.webhook.") { + t.Fatal("expected named subconfig to be detected as enabled") + } + + targets, err := buildTargets(cfg, "notification.webhook.") + if err != nil { + t.Fatalf("buildTargets() error = %v", err) + } + if len(targets) != 2 { + t.Fatalf("expected 2 targets, got %d", len(targets)) + } + if targets[0].name != "file_ingress" || targets[1].name != "metadata_extractor" { + t.Fatalf("unexpected target names: %s, %s", targets[0].name, targets[1].name) + } + + if err := q.initialize(targets); err != nil { + t.Fatalf("initialize() error = %v", err) + } + defer closeQueueForTest(q) + + time.Sleep(200 * time.Millisecond) + + msg := &filer_pb.EventNotification{ + NewEntry: &filer_pb.Entry{Name: "f.txt"}, + } + for _, key := range []string{"/data/f.txt", "/repository/f.txt", "/elsewhere/f.txt"} { + if err := q.SendMessage(key, msg); err != nil { + t.Errorf("SendMessage(%q) error = %v", key, err) + } + } + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && (hitsA.Load() == 0 || hitsB.Load() == 0) { + time.Sleep(50 * time.Millisecond) + } + time.Sleep(200 * time.Millisecond) + + if got := hitsA.Load(); got != 1 { + t.Errorf("expected file_ingress to receive 1 event, got %d", got) + } + if got := hitsB.Load(); got != 1 { + t.Errorf("expected metadata_extractor to receive 1 event, got %d", got) + } +} + +func TestQueueBurstKeepsPendingEvents(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + time.Sleep(20 * time.Millisecond) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + cfg := &config{ + endpoint: server.URL, + timeoutSeconds: 5, + maxRetries: 0, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 2, + } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) + + const total = 6 + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + for i := 0; i < total; i++ { + if err := q.SendMessage("/test/path", msg); err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + } + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) && hits.Load() < total { + time.Sleep(50 * time.Millisecond) + } + if got := hits.Load(); got != total { + t.Errorf("expected %d deliveries, got %d", total, got) + } +} + +// A saturated target still applies backpressure to the sender, but it must +// not starve the targets after it: the event reaches a healthy endpoint +// while the slow one is still out of pending slots. +func TestSaturatedTargetDoesNotStarveOthers(t *testing.T) { + var hitsA, hitsB atomic.Int32 + serverA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsA.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverA.Close() + serverB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hitsB.Add(1) + w.WriteHeader(http.StatusOK) + })) + defer serverB.Close() + + newTestTarget := func(name, endpoint string) *target { + tgt, err := newTarget(name, &config{ + endpoint: endpoint, + timeoutSeconds: 5, + maxRetries: 0, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 2, + }) + if err != nil { + t.Fatalf("newTarget() error = %v", err) + } + return tgt + } + slow := newTestTarget("slow", serverA.URL) + fast := newTestTarget("fast", serverB.URL) + + q := &Queue{} + if err := q.initialize([]*target{slow, fast}); err != nil { + t.Fatalf("initialize() error = %v", err) + } + defer closeQueueForTest(q) + + for i := 0; i < cap(slow.pending); i++ { + slow.pending <- struct{}{} + } + + done := make(chan error, 1) + go func() { + done <- q.SendMessage("/test/path", &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}}) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && hitsB.Load() == 0 { + time.Sleep(20 * time.Millisecond) + } + if got := hitsB.Load(); got != 1 { + t.Fatalf("expected fast target to receive the event while slow is saturated, got %d", got) + } + select { + case err := <-done: + t.Fatalf("SendMessage() returned %v while the saturated target had no free slot", err) + default: + } + + <-slow.pending + if err := <-done; err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + + deadline = time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) && hitsA.Load() == 0 { + time.Sleep(20 * time.Millisecond) + } + if got := hitsA.Load(); got != 1 { + t.Errorf("expected the saturated target to deliver once a slot freed, got %d", got) + } +} + +// A disabled flat section keeps no target even when an endpoint is set — +// the scaffold ships exactly that combination, and a named target must not +// reactivate it. +func TestBuildTargetsDisabledFlat(t *testing.T) { + cfg := &mapConfig{values: map[string]interface{}{ + "notification.webhook": map[string]interface{}{ + "enabled": false, + "endpoint": "https://your-server.com/webhook", + "named": map[string]interface{}{ + "enabled": true, + "endpoint": "https://example.com/hook", + }, + }, + "notification.webhook.enabled": false, + "notification.webhook.endpoint": "https://your-server.com/webhook", + "notification.webhook.named.enabled": true, + "notification.webhook.named.endpoint": "https://example.com/hook", + }} + + targets, err := buildTargets(cfg, "notification.webhook.") + if err != nil { + t.Fatalf("buildTargets() error = %v", err) + } + if len(targets) != 1 || targets[0].name != "named" { + t.Fatalf("expected only the named target, got %v", targets) + } +} + +// A failing endpoint must be retried maxRetries times before the event is +// dead-lettered — the poison middleware has to sit outside the retry chain +// or it swallows the first error and retries never run. +func TestQueueRetriesBeforeDeadLetter(t *testing.T) { + var hits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + cfg := &config{ + endpoint: server.URL, + timeoutSeconds: 5, + maxRetries: 2, + backoffSeconds: 1, + maxBackoffSeconds: 1, + nWorkers: 1, + bufferSize: 10, + } + q := newQueueForTest(t, cfg) + defer closeQueueForTest(q) + + msg := &filer_pb.EventNotification{NewEntry: &filer_pb.Entry{Name: "f.txt"}} + if err := q.SendMessage("/test/path", msg); err != nil { + t.Fatalf("SendMessage() error = %v", err) + } + + want := int32(cfg.maxRetries + 1) + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) && hits.Load() < want { + time.Sleep(50 * time.Millisecond) + } + if got := hits.Load(); got != want { + t.Errorf("expected %d delivery attempts (1 + %d retries), got %d", want, cfg.maxRetries, got) + } +} From 689ea6ea3e2bb1beeb87f53cfbf1b6747f16c17a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 06:47:24 -0700 Subject: [PATCH 50/70] release: digest-bound provenance and SBOM attestations for container images (#12024) * release: emit digest-bound provenance and SBOM attestations Per-platform builds now record SLSA provenance at mode=max and a BuildKit SBOM, so the builder identity and resolved dependencies are captured instead of the empty predicate the 4.48 build record carried. After the multi-arch index is assembled and signed, the merge job publishes GitHub artifact attestations bound to the merged digest and a syft SPDX SBOM of the image, giving digest-pinning consumers a verifiable record on the exact index they pull. * release: keep unfixed findings in the trivy SARIF upload ignore-unfixed dropped every finding without an available patch, so the uploaded report understated the image's exposure. The scan stays report-only; only the completeness of what is uploaded changes. * release: generate and attest SBOMs per platform digest An index-wide syft scan only describes its default platform, so the SBOM attested to the merged multi-arch digest misrepresented every other manifest. Scan and attest each platform digest instead, which also drops the sbom-action upload defaults that raced release-asset permissions and artifact names across the merge matrix. * release: resolve per-platform index to its image manifest before syft/cosign push-by-digest records point at OCI indexes (image + attestation manifests), so scanning or attesting the index digest picks the runner's platform and fails outright for the other arches under set -e. Resolve the non-attestation manifest inside each index and bind the SBOM attestation to the image digest a same-platform pull of the merged index actually returns. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../workflows/container_release_unified.yml | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 3080595eb4..320436cb2e 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -271,8 +271,11 @@ jobs: file: ${{ matrix.dockerfile }} platforms: ${{ matrix.platform }} labels: ${{ steps.docker_meta.outputs.labels }} - # Flat single-platform manifest so imagetools create assembles cleanly. - provenance: false + # mode=max keeps resolvedDependencies complete and records the + # builder identity; attestations ride inside each pushed index and + # survive the imagetools merge into the multi-arch manifest. + provenance: mode=max + sbom: true outputs: type=image,name=${{ env.IMAGE }},push-by-digest=true,name-canonical=true,push=true cache-from: type=gha,scope=${{ matrix.variant }}-${{ matrix.arch }} # max only for rocksdb: its RocksDB compile is sha-independent and worth @@ -308,6 +311,8 @@ jobs: permissions: contents: read id-token: write + attestations: write + packages: write strategy: fail-fast: false matrix: @@ -415,6 +420,45 @@ jobs: ${{ env.IMAGE }}@${{ steps.manifest.outputs.digest }} chrislusf/seaweedfs@${{ steps.manifest.outputs.digest }} + # Attest the merged multi-arch digest itself, so digest-pinning consumers + # can verify provenance against the exact index they pull. Both steps + # reuse the GHCR login above and push as OCI referrers. + - name: Attest build provenance for ${{ matrix.variant }} + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: actions/attest-build-provenance@v4 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ steps.manifest.outputs.digest }} + push-to-registry: true + + # SBOMs are generated and attested per platform manifest: an index-wide + # scan only describes syft's default platform, while the manifests it + # merges differ (some ship the real binary, others placeholders), so the + # SBOM has to be bound to the platform digest a consumer actually pulls. + - name: Install syft + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: anchore/sbom-action/download-syft@v0 + + - name: Attest per-platform SBOMs for ${{ matrix.variant }} + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + working-directory: /tmp/digests + run: | + set -euo pipefail + for f in *; do + # Each saved digest is an OCI index (image + attestation manifests); + # scan and attest the image manifest a same-platform pull resolves. + ref="${{ env.IMAGE }}@sha256:${f}" + image=$(crane manifest "$ref" | jq -r ' + [.manifests[]? | select((.annotations["vnd.docker.reference.type"] // "") != "attestation-manifest" + and (.platform.os // "unknown") != "unknown") | .digest][0] // empty') + ref="${{ env.IMAGE }}@${image:-sha256:${f}}" + syft "registry:${ref}" -o "spdx-json=/tmp/sbom-${f}.spdx.json" + cosign attest --yes \ + --predicate "/tmp/sbom-${f}.spdx.json" \ + --type spdxjson \ + "$ref" + done + # Report-only trivy scan: uploads fixable HIGH/CRITICAL findings to GitHub # Security for visibility, but never blocks the release. Releases (including # `latest`) ship regardless — vulnerabilities are tracked, not gated, since @@ -453,7 +497,9 @@ jobs: scanners: vuln vuln-type: os,library severity: HIGH,CRITICAL - ignore-unfixed: true + # Keep unfixed findings in the report so digest-pinning consumers see + # the complete picture, not just what a patch could resolve. + ignore-unfixed: false limit-severities-for-sarif: true format: sarif output: trivy-results.sarif From fd3b437e04d55136e0d780d44402dc4709c4a98f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 06:57:02 -0700 Subject: [PATCH 51/70] s3api: write the null version's object-lock metadata to the plain entry (#12033) * s3api: write the null version's lock metadata to the plain entry PutObjectRetention and PutObjectLegalHold with versionId=null loaded the pre-versioning entry but wrote the update under .versions/v_null, so the acknowledged lock never bound the object it was read from. Resolve the entry and its write path in one helper that applies the same null-version rule as getSpecificObjectVersion. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: cover the null-version lock write path Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3api: persist object-lock metadata with an in-place entry update mkFile rebuilds the target entry from chunks alone, so a lock write on a plain (null-version) entry discarded attributes, inline content, and the remote reference. UpdateEntry keeps the whole fetched entry, matching how the tagging and ACL writers persist metadata. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/s3api_object_retention.go | 163 ++++++++-------------- weed/s3api/s3api_object_retention_test.go | 100 +++++++++++++ 2 files changed, 160 insertions(+), 103 deletions(-) diff --git a/weed/s3api/s3api_object_retention.go b/weed/s3api/s3api_object_retention.go index 2d1e8f4ff2..53f92f3574 100644 --- a/weed/s3api/s3api_object_retention.go +++ b/weed/s3api/s3api_object_retention.go @@ -12,6 +12,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/s3api/s3err" + "github.com/seaweedfs/seaweedfs/weed/util" ) // ==================================================================== @@ -213,6 +214,49 @@ func (s3a *S3ApiServer) getObjectEntry(bucket, object, versionId string) (*filer return entry, nil } +// getObjectEntryForUpdate resolves the entry a metadata update targets and the +// filer path it is written under. It applies the same version resolution as +// getSpecificObjectVersion - "null" names the plain pre-versioning entry, not a +// .versions file - so the write lands on the entry that was read. +func (s3a *S3ApiServer) getObjectEntryForUpdate(bucket, object, versionId string) (entry *filer_pb.Entry, entryPath string, err error) { + entryPath = object + if versionId != "" { + entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) + if err != nil { + return nil, "", fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) + } + if versionId != "null" { + entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) + } + return entry, entryPath, nil + } + + versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) + if vErr != nil { + return nil, "", fmt.Errorf("error checking versioning: %w", vErr) + } + if !versioningEnabled { + entry, err = s3a.fetchObjectEntryRequired(bucket, object) + if err != nil { + return nil, "", fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) + } + return entry, entryPath, nil + } + + entry, err = s3a.getLatestObjectVersion(bucket, object) + if err != nil { + return nil, "", fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) + } + if entry.Extended != nil { + if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { + if storedVersionId := string(versionIdBytes); storedVersionId != "null" && isValidVersionID(storedVersionId) { + entryPath = object + ".versions/" + s3a.getVersionFileName(storedVersionId) + } + } + } + return entry, entryPath, nil +} + // ==================================================================== // RETENTION OPERATIONS // ==================================================================== @@ -254,47 +298,9 @@ func (s3a *S3ApiServer) getObjectRetention(bucket, object, versionId string) (*O // setObjectRetention sets object retention configuration func (s3a *S3ApiServer) setObjectRetention(bucket, object, versionId string, retention *ObjectRetention, bypassGovernance bool) error { - var entry *filer_pb.Entry - var err error - var entryPath string - - if versionId != "" { - entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) - if err != nil { - return fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) - } - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } else { - // Check if versioning is enabled - versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) - if vErr != nil { - return fmt.Errorf("error checking versioning: %w", vErr) - } - - if versioningEnabled { - entry, err = s3a.getLatestObjectVersion(bucket, object) - if err != nil { - return fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) - } - // Extract version ID from entry metadata - entryPath = object // default to regular object path - if entry.Extended != nil { - if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { - versionId = string(versionIdBytes) - // A stored version id must be a valid path segment before it can - // name a .versions file; otherwise fall back to the regular object. - if versionId != "null" && isValidVersionID(versionId) { - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } - } - } - } else { - entry, err = s3a.fetchObjectEntryRequired(bucket, object) - if err != nil { - return fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) - } - entryPath = object - } + entry, entryPath, err := s3a.getObjectEntryForUpdate(bucket, object, versionId) + if err != nil { + return err } // Check if object is already under retention @@ -355,18 +361,15 @@ func (s3a *S3ApiServer) setObjectRetention(bucket, object, versionId string, ret entry.WormEnforcedAtTsNs = time.Now().UnixNano() } - // Update the entry + // Update the entry in place. UpdateEntry persists the whole fetched entry, so + // attributes, inline content, and remote references survive the metadata change + // where mkFile's fresh entry would drop them. // NOTE: Potential race condition exists if concurrent calls to PutObjectRetention // and PutObjectLegalHold update the same object simultaneously, as they might - // overwrite each other's Extended map changes. This is mitigated by the fact - // that mkFile operations are typically serialized at the filer level, but - // future implementations might consider using atomic update operations or - // entry-level locking for complete safety. - bucketDir := s3a.bucketDir(bucket) - return s3a.mkFile(bucketDir, entryPath, entry.Chunks, func(updatedEntry *filer_pb.Entry) { - updatedEntry.Extended = entry.Extended - updatedEntry.WormEnforcedAtTsNs = entry.WormEnforcedAtTsNs - }) + // overwrite each other's Extended map changes. + dir, name := util.NewFullPath(s3a.bucketDir(bucket), entryPath).DirAndName() + entry.Name = name + return s3a.updateEntry(dir, entry) } // ==================================================================== @@ -399,47 +402,9 @@ func (s3a *S3ApiServer) getObjectLegalHold(bucket, object, versionId string) (*O // setObjectLegalHold sets object legal hold configuration func (s3a *S3ApiServer) setObjectLegalHold(bucket, object, versionId string, legalHold *ObjectLegalHold) error { - var entry *filer_pb.Entry - var err error - var entryPath string - - if versionId != "" { - entry, err = s3a.getSpecificObjectVersion(bucket, object, versionId) - if err != nil { - return fmt.Errorf("failed to get version %s for object %s/%s: %w", versionId, bucket, object, ErrVersionNotFound) - } - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } else { - // Check if versioning is enabled - versioningEnabled, vErr := s3a.isVersioningEnabled(bucket) - if vErr != nil { - return fmt.Errorf("error checking versioning: %w", vErr) - } - - if versioningEnabled { - entry, err = s3a.getLatestObjectVersion(bucket, object) - if err != nil { - return fmt.Errorf("failed to get latest version for object %s/%s: %w", bucket, object, ErrLatestVersionNotFound) - } - // Extract version ID from entry metadata - entryPath = object // default to regular object path - if entry.Extended != nil { - if versionIdBytes, exists := entry.Extended[s3_constants.ExtVersionIdKey]; exists { - versionId = string(versionIdBytes) - // A stored version id must be a valid path segment before it can - // name a .versions file; otherwise fall back to the regular object. - if versionId != "null" && isValidVersionID(versionId) { - entryPath = object + ".versions/" + s3a.getVersionFileName(versionId) - } - } - } - } else { - entry, err = s3a.fetchObjectEntryRequired(bucket, object) - if err != nil { - return fmt.Errorf("failed to get object %s/%s: %w", bucket, object, ErrObjectNotFound) - } - entryPath = object - } + entry, entryPath, err := s3a.getObjectEntryForUpdate(bucket, object, versionId) + if err != nil { + return err } // Update legal hold metadata @@ -449,17 +414,9 @@ func (s3a *S3ApiServer) setObjectLegalHold(bucket, object, versionId string, leg entry.Extended[s3_constants.ExtLegalHoldKey] = []byte(legalHold.Status) - // Update the entry - // NOTE: Potential race condition exists if concurrent calls to PutObjectRetention - // and PutObjectLegalHold update the same object simultaneously, as they might - // overwrite each other's Extended map changes. This is mitigated by the fact - // that mkFile operations are typically serialized at the filer level, but - // future implementations might consider using atomic update operations or - // entry-level locking for complete safety. - bucketDir := s3a.bucketDir(bucket) - return s3a.mkFile(bucketDir, entryPath, entry.Chunks, func(updatedEntry *filer_pb.Entry) { - updatedEntry.Extended = entry.Extended - }) + dir, name := util.NewFullPath(s3a.bucketDir(bucket), entryPath).DirAndName() + entry.Name = name + return s3a.updateEntry(dir, entry) } // ==================================================================== diff --git a/weed/s3api/s3api_object_retention_test.go b/weed/s3api/s3api_object_retention_test.go index 44089deb33..201d471658 100644 --- a/weed/s3api/s3api_object_retention_test.go +++ b/weed/s3api/s3api_object_retention_test.go @@ -1,15 +1,23 @@ package s3api import ( + "context" "fmt" "io" "net/http" "strings" + "sync" "testing" "time" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/protobuf/proto" + + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/seaweedfs/seaweedfs/weed/wdclient" ) func TestValidateRetention(t *testing.T) { @@ -857,3 +865,95 @@ func TestStoreObjectLockConfigurationClearsStaleYears(t *testing.T) { func timePtr(t time.Time) *time.Time { return &t } + +// retentionTestFiler serves the two filer calls a lock write needs: the lookup +// that resolves the version and the update that persists the modified entry. +type retentionTestFiler struct { + filer_pb.UnimplementedSeaweedFilerServer + + mu sync.Mutex + entries map[string]*filer_pb.Entry + updates []string +} + +func (f *retentionTestFiler) LookupDirectoryEntry(_ context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + if entry, ok := f.entries[req.Directory+"/"+req.Name]; ok { + return &filer_pb.LookupDirectoryEntryResponse{Entry: proto.Clone(entry).(*filer_pb.Entry)}, nil + } + return &filer_pb.LookupDirectoryEntryResponse{}, nil +} + +func (f *retentionTestFiler) UpdateEntry(_ context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.updates = append(f.updates, req.Directory+"/"+req.Entry.Name) + f.entries[req.Directory+"/"+req.Entry.Name] = req.Entry + return &filer_pb.UpdateEntryResponse{}, nil +} + +func newRetentionTestServer(t *testing.T, f *retentionTestFiler) *S3ApiServer { + t.Helper() + addr := startFakeFiler(t, f) + dialOption := grpc.WithTransportCredentials(insecure.NewCredentials()) + return &S3ApiServer{ + option: &S3ApiServerOption{ + Filers: []pb.ServerAddress{addr}, + GrpcDialOption: dialOption, + BucketsPath: "/buckets", + }, + filerClient: wdclient.NewFilerClient([]pb.ServerAddress{addr}, dialOption, ""), + } +} + +// A "null" version id resolves to the pre-versioning entry under the plain +// object key, the same place getSpecificObjectVersion reads it from. Writing +// the lock under .versions/v_null leaves the real entry unlocked. +func TestNullVersionLockWritesToPlainEntry(t *testing.T) { + f := &retentionTestFiler{entries: map[string]*filer_pb.Entry{ + "/buckets/b/key": { + Name: "key", + Chunks: []*filer_pb.FileChunk{{FileId: "1,0000000000000001"}}, + Content: []byte("inline"), + Attributes: &filer_pb.FuseAttributes{ + Mime: "text/plain", + Md5: []byte("etagmd5"), + }, + Extended: map[string][]byte{ + "ETag": []byte("\"etagmd5\""), + }, + }, + }} + s3a := newRetentionTestServer(t, f) + + retainUntil := time.Now().Add(24 * time.Hour).Truncate(time.Second) + err := s3a.setObjectRetention("b", "key", "null", &ObjectRetention{ + Mode: s3_constants.RetentionModeCompliance, + RetainUntilDate: &retainUntil, + }, false) + if err != nil { + t.Fatalf("setObjectRetention: %v", err) + } + if len(f.updates) != 1 || f.updates[0] != "/buckets/b/key" { + t.Fatalf("retention write landed on %v, want /buckets/b/key", f.updates) + } + updated := f.entries["/buckets/b/key"] + if got := string(updated.Extended[s3_constants.ExtObjectLockModeKey]); got != s3_constants.RetentionModeCompliance { + t.Fatalf("plain entry lock mode = %q, want COMPLIANCE", got) + } + if updated.Attributes.Mime != "text/plain" || len(updated.Content) == 0 || len(updated.Chunks) != 1 || string(updated.Extended["ETag"]) != "\"etagmd5\"" { + t.Fatalf("lock write dropped entry fields: %+v", updated) + } + + f.updates = nil + if err := s3a.setObjectLegalHold("b", "key", "null", &ObjectLegalHold{Status: "ON"}); err != nil { + t.Fatalf("setObjectLegalHold: %v", err) + } + if len(f.updates) != 1 || f.updates[0] != "/buckets/b/key" { + t.Fatalf("legal hold write landed on %v, want /buckets/b/key", f.updates) + } + if got := string(f.entries["/buckets/b/key"].Extended[s3_constants.ExtLegalHoldKey]); got != "ON" { + t.Fatalf("plain entry legal hold = %q, want ON", got) + } +} From 9281ed80070e5d9a65c4b837ca0eca6e23e53b45 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 06:58:16 -0700 Subject: [PATCH 52/70] master: never auto-admit raft peers on a single-node configuration (#12021) * master: never auto-admit raft peers on a single-node configuration A master started without cluster peers (peers=none, or no peers so the list holds only itself) declares a one-node cluster, yet any foreign master that registered over KeepConnected was admitted as a second voter. The resulting two-voter cluster cannot reach quorum when the foreign peer is unreachable, wedging a previously healthy single master. Gate raft admission on having been configured with cluster peers; masters started with a peer list keep the announce-to-join flow used to grow the cluster. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * master: source SingleMaster from -peers=none, not the peer count A master started with an empty peer list ends up with a one-peer map containing itself, and NewMasterServer inferred declared single-master mode from that count. The master still mints a raft cluster through the bootstrap path, but AdmitRaftPeer then ignores every later joiner, so a deployment that grows from one master to several can never form a quorum. isSingleMasterMode already spells out the declared case (-peers=none); populate MasterOption.SingleMaster from it instead. * master: cover clustered raft admission with a real in-memory leader The previous clustered test had no raft leader and asserted nothing, so an AdmitRaftPeer that always returned early still passed. Bootstrap a single-node hashicorp raft over the in-memory transport, wait for leadership, and assert the leader reports and admits a missing peer. * master: return the raft admission result and assert committed join AdmitRaftPeer ran AddVoter on a detached goroutine, so the only observable state was an in-flight marker deleted on success and failure alike, and the clustered test could never tell whether the join worked. Admission now hands back a result channel; the test runs a connected second in-memory node and asserts the peer sees itself a committed voter. --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/master.go | 1 + weed/server/master_server.go | 23 +++-- weed/server/master_server_admit_test.go | 119 ++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 weed/server/master_server_admit_test.go diff --git a/weed/command/master.go b/weed/command/master.go index b12f0a6210..d205c3a6a9 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -491,5 +491,6 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp MetricsIntervalSec: *m.metricsIntervalSec, TelemetryUrl: *m.telemetryUrl, TelemetryEnabled: *m.telemetryEnabled, + SingleMaster: isSingleMasterMode(*m.peers), } } diff --git a/weed/server/master_server.go b/weed/server/master_server.go index dec88ab154..1e6cb58df9 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -66,6 +66,7 @@ type MasterOption struct { MetricsAddress string MetricsIntervalSec int IsFollower bool + SingleMaster bool TelemetryUrl string TelemetryEnabled bool VolumeGrowthDisabled bool @@ -106,7 +107,6 @@ type MasterServer struct { } func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer { - v := util.GetViper() signingKey := v.GetString("jwt.signing.key") v.SetDefault("jwt.signing.expires_after_seconds", 10) @@ -601,18 +601,20 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF // AdmitRaftPeer pulls a master into the raft quorum, if we are the leader and // do not have it yet. A master that starts with no raft state cannot campaign -// under either implementation, so this is its only way in. -func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) { - if peerAddress.ToHttpAddress() == ms.option.Master.ToHttpAddress() { - return +// under either implementation, so this is its only way in. The returned +// channel resolves with the admission result, or nil when no admission ran. +func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) <-chan error { + if ms.option.SingleMaster || peerAddress.ToHttpAddress() == ms.option.Master.ToHttpAddress() { + return nil } peerName, ok := ms.missingRaftPeerName(peerAddress) if !ok { - return + return nil } - if _, alreadyAdmitting := ms.raftPeerAdmissions.LoadOrStore(peerName, struct{}{}); alreadyAdmitting { - return + admission := make(chan error, 1) + if inFlight, alreadyAdmitting := ms.raftPeerAdmissions.LoadOrStore(peerName, admission); alreadyAdmitting { + return inFlight.(chan error) } glog.V(0).Infof("adding new raft server: %s", peerName) // The join commits through raft, which waits on the other peers, so keep it @@ -620,10 +622,15 @@ func (ms *MasterServer) AdmitRaftPeer(peerAddress pb.ServerAddress) { // handler that a joining master is still blocked in. go func() { defer ms.raftPeerAdmissions.Delete(peerName) + defer close(admission) if err := ms.raftAddServer(peerName, peerAddress.ToGrpcAddress(), true); err != nil { glog.Warningf("failed adding raft server %s: %v", peerName, err) + admission <- err + return } + admission <- nil }() + return admission } // missingRaftPeerName returns the raft id to admit peerAddress under, and diff --git a/weed/server/master_server_admit_test.go b/weed/server/master_server_admit_test.go new file mode 100644 index 0000000000..012985b5cf --- /dev/null +++ b/weed/server/master_server_admit_test.go @@ -0,0 +1,119 @@ +package weed_server + +import ( + "testing" + "time" + + hashicorpRaft "github.com/hashicorp/raft" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/topology" +) + +func TestSingleMasterNeverAdmitsRaftPeers(t *testing.T) { + ms := &MasterServer{option: &MasterOption{SingleMaster: true, Master: "localhost:9333"}} + + // A foreign master announcing itself must not touch the admission path: + // ms.Topo is deliberately nil so any admission attempt panics the test. + if done := ms.AdmitRaftPeer("10.0.0.1:9333"); done != nil { + t.Fatal("single-master mode admitted a foreign raft peer") + } +} + +func TestClusteredLeaderAdmitsMissingPeer(t *testing.T) { + topo := topology.NewTopology("topo", nil, 128*1024*1024, 0, false) + leader, leaderTransport := newInmemRaftLeader(t, "localhost:9333") + topo.HashicorpRaft = leader + ms := &MasterServer{ + option: &MasterOption{Master: "localhost:9333"}, + Topo: topo, + } + + peer := pb.ServerAddress("10.0.0.1:9333") + name, missing := ms.missingRaftPeerName(peer) + if !missing || name != "10.0.0.1:9333" { + t.Fatalf("clustered leader did not report the missing raft peer: %q %v", name, missing) + } + peerRaft := newInmemRaftFollower(t, peer, leaderTransport) + + done := ms.AdmitRaftPeer(peer) + if done == nil { + t.Fatal("clustered leader skipped admission of a missing raft peer") + } + if err := <-done; err != nil { + t.Fatalf("raft peer admission failed: %v", err) + } + + // The joined node's own configuration is the committed view: it only + // learns the quorum it was admitted into through replication. + for deadline := time.Now().Add(10 * time.Second); ; { + if raftHasVoter(peerRaft, string(peer)) { + return + } + if time.Now().After(deadline) { + t.Fatalf("admitted peer never saw itself in the quorum: %v", + peerRaft.GetConfiguration().Configuration().Servers) + } + time.Sleep(10 * time.Millisecond) + } +} + +func raftHasVoter(r *hashicorpRaft.Raft, id string) bool { + for _, server := range r.GetConfiguration().Configuration().Servers { + if server.Suffrage == hashicorpRaft.Voter && string(server.ID) == id { + return true + } + } + return false +} + +func newInmemRaftFollower(t *testing.T, self pb.ServerAddress, peerTransport *hashicorpRaft.InmemTransport) *hashicorpRaft.Raft { + t.Helper() + conf := hashicorpRaft.DefaultConfig() + conf.LocalID = hashicorpRaft.ServerID(self) + addr, transport := hashicorpRaft.NewInmemTransport(hashicorpRaft.ServerAddress(self.ToGrpcAddress())) + transport.Connect(peerTransport.LocalAddr(), peerTransport) + peerTransport.Connect(addr, transport) + r, err := hashicorpRaft.NewRaft(conf, + &StateMachine{topo: topology.NewTopology("fsm", nil, 128*1024*1024, 0, false)}, + hashicorpRaft.NewInmemStore(), hashicorpRaft.NewInmemStore(), + hashicorpRaft.NewInmemSnapshotStore(), transport) + if err != nil { + t.Fatalf("new raft: %v", err) + } + t.Cleanup(func() { r.Shutdown() }) + return r +} + +func newInmemRaftLeader(t *testing.T, self string) (*hashicorpRaft.Raft, *hashicorpRaft.InmemTransport) { + t.Helper() + conf := hashicorpRaft.DefaultConfig() + conf.LocalID = hashicorpRaft.ServerID(self) + conf.HeartbeatTimeout = 50 * time.Millisecond + conf.ElectionTimeout = 50 * time.Millisecond + conf.LeaderLeaseTimeout = 25 * time.Millisecond + conf.CommitTimeout = 5 * time.Millisecond + addr, transport := hashicorpRaft.NewInmemTransport(hashicorpRaft.ServerAddress(self)) + r, err := hashicorpRaft.NewRaft(conf, + &StateMachine{topo: topology.NewTopology("fsm", nil, 128*1024*1024, 0, false)}, + hashicorpRaft.NewInmemStore(), hashicorpRaft.NewInmemStore(), + hashicorpRaft.NewInmemSnapshotStore(), transport) + if err != nil { + t.Fatalf("new raft: %v", err) + } + t.Cleanup(func() { r.Shutdown() }) + + if err := r.BootstrapCluster(hashicorpRaft.Configuration{ + Servers: []hashicorpRaft.Server{{Suffrage: hashicorpRaft.Voter, ID: conf.LocalID, Address: addr}}, + }).Error(); err != nil { + t.Fatalf("bootstrap cluster: %v", err) + } + for deadline := time.Now().Add(10 * time.Second); ; { + if r.State() == hashicorpRaft.Leader { + return r, transport + } + if time.Now().After(deadline) { + t.Fatal("in-memory raft did not elect a leader") + } + time.Sleep(10 * time.Millisecond) + } +} From 5700aa5e91374f9d76f40b455e6c9a4488125ca5 Mon Sep 17 00:00:00 2001 From: Wleeaf Date: Sat, 24 Oct 2026 16:58:59 +0300 Subject: [PATCH 53/70] s3: evaluate bucket CORS for path-style OPTIONS on the bucket root (#12028) * s3: evaluate bucket CORS for path-style OPTIONS on the bucket root A path-style preflight for /bucket (no trailing slash) missed the bucket OPTIONS route, whose template is /{bucket}/, and was answered by the global OPTIONS handler from the global AllowedOrigins instead of the bucket CORS configuration. Register the bucket OPTIONS route without a path matcher, and keep OPTIONS for the health probe paths on the service-level handler ahead of each path-style router. Fixes #12025 * test: drop the issue reference from the bucket-root preflight test comment --------- Co-authored-by: Wleeaf <179125857+wleeaf@users.noreply.github.com> Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/s3api_server.go | 22 +++++-- weed/s3api/s3api_server_routing_test.go | 87 +++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 5 deletions(-) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index c10e311e3d..dc9cb139a2 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -947,6 +947,17 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // Object path pattern with (?s) flag to match newlines in object keys const objectPath = "/{object:(?s).+}" + // CORS preflight for service-level requests, answered from the global + // AllowedOrigins rather than a bucket's CORS configuration + serviceOptionsHandler := func(w http.ResponseWriter, r *http.Request) { + if s3a.handleCORSOriginValidation(w, r) { + writeSuccessResponseEmpty(w, r) + } + } + // A path-style bucket router would take the health probe paths as bucket + // names, so their preflight is registered ahead of each one + const healthProbePath = "/{probe:status|healthz|readyz}" + var routers []*mux.Router if s3a.option.DomainName != "" { domainNames := strings.Split(s3a.option.DomainName, ",") @@ -954,6 +965,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // Register path-style domains for _, domain := range pathStyleDomains { + apiRouter.Host(domain).Methods(http.MethodOptions).Path(healthProbePath).HandlerFunc(serviceOptionsHandler) routers = append(routers, apiRouter.Host(domain).PathPrefix("/{bucket}").Subrouter()) apiRouter.Host(domain). Methods(http.MethodGet). @@ -971,6 +983,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { // arrive via an IP, an unlisted hostname, or a reverse proxy that rewrites // the Host header still match bucket routes. Host-specific routers above // take precedence because they were registered first. + apiRouter.Methods(http.MethodOptions).Path(healthProbePath).HandlerFunc(serviceOptionsHandler) routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) // Get CORS middleware instance with caching @@ -986,8 +999,9 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Use(corsMiddleware.Handler) // Bucket-specific OPTIONS handler for CORS preflight requests - // Use PathPrefix to catch all bucket-level preflight routes including /bucket/object - bucket.PathPrefix("/").Methods(http.MethodOptions).HandlerFunc(corsMiddleware.HandleOptionsRequest) + // No path matcher, so it also catches the path-style bucket root (/bucket), + // which PathPrefix("/") would miss, as well as /bucket/ and /bucket/object + bucket.Methods(http.MethodOptions).HandlerFunc(corsMiddleware.HandleOptionsRequest) // each case should follow the next rule: // - requesting object with query must precede any other methods @@ -1222,9 +1236,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { return } - if s3a.handleCORSOriginValidation(w, r) { - writeSuccessResponseEmpty(w, r) - } + serviceOptionsHandler(w, r) }) // STS API endpoint for AssumeRoleWithWebIdentity diff --git a/weed/s3api/s3api_server_routing_test.go b/weed/s3api/s3api_server_routing_test.go index 9459fe588f..9316174924 100644 --- a/weed/s3api/s3api_server_routing_test.go +++ b/weed/s3api/s3api_server_routing_test.go @@ -17,6 +17,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/iam/policy" "github.com/seaweedfs/seaweedfs/weed/iam/sts" "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/cors" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/util" "github.com/stretchr/testify/assert" @@ -442,3 +443,89 @@ func TestRouting_BucketRouteMatchesAnyHost(t *testing.T) { }) } } + +// TestRouting_BucketRootPreflightUsesBucketCORS is a regression test: a +// path-style preflight for the bucket root without a trailing +// slash must be evaluated against the bucket CORS rules like /bucket/, not +// answered from the global AllowedOrigins. Service-level paths keep the +// global answer, and virtual-host routing is unchanged. +func TestRouting_BucketRootPreflightUsesBucketCORS(t *testing.T) { + const ( + bucket = "cors-bucket" + allowedOrigin = "https://app.example.com" + foreignOrigin = "https://untrusted.example" + ) + pathStyleBucketPaths := []string{"/" + bucket, "/" + bucket + "/", "/" + bucket + "?list-type=2", "/" + bucket + "/key"} + + cases := []struct { + name string + domainName string + host string + bucketPaths []string // evaluated against the bucket CORS rules + servicePaths []string // answered from the global AllowedOrigins + }{ + { + name: "path-style", + host: "10.0.0.5:8333", + bucketPaths: pathStyleBucketPaths, + servicePaths: []string{"/", "/status", "/healthz", "/readyz"}, + }, + { + name: "path-style domain", + domainName: "s3.example.com,api.s3.example.com", + host: "api.s3.example.com", + bucketPaths: pathStyleBucketPaths, + servicePaths: []string{"/status", "/healthz", "/readyz"}, + }, + { + name: "virtual-host domain", + domainName: "s3.example.com,api.s3.example.com", + host: bucket + ".s3.example.com", + bucketPaths: []string{"/", "/?list-type=2", "/key", "/status"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + s3a := setupRoutingTestServer(t) + s3a.option.DomainName = tc.domainName + s3a.option.AllowedOrigins = []string{"*"} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set(bucket, &BucketConfig{ + Name: bucket, + CORS: &cors.CORSConfiguration{CORSRules: []cors.CORSRule{{ + AllowedOrigins: []string{allowedOrigin}, + AllowedMethods: []string{http.MethodGet, http.MethodPut}, + AllowedHeaders: []string{"*"}, + }}}, + }) + router := mux.NewRouter() + s3a.registerRouter(router) + + preflight := func(path, origin string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodOptions, "http://"+tc.host+path, nil) + req.Header.Set("Origin", origin) + req.Header.Set("Access-Control-Request-Method", http.MethodPut) + rr := httptest.NewRecorder() + router.ServeHTTP(rr, req) + return rr + } + + for _, path := range tc.bucketPaths { + rr := preflight(path, foreignOrigin) + assert.Equal(t, http.StatusForbidden, rr.Code, "foreign origin on %s", path) + assert.Empty(t, rr.Header().Get("Access-Control-Allow-Origin"), "foreign origin on %s", path) + + rr = preflight(path, allowedOrigin) + assert.Equal(t, http.StatusOK, rr.Code, "allowed origin on %s", path) + assert.Equal(t, allowedOrigin, rr.Header().Get("Access-Control-Allow-Origin"), "allowed origin on %s", path) + } + + for _, path := range tc.servicePaths { + rr := preflight(path, foreignOrigin) + assert.Equal(t, http.StatusOK, rr.Code, "service-level %s", path) + assert.Equal(t, "*", rr.Header().Get("Access-Control-Allow-Origin"), "service-level %s", path) + } + }) + } +} From 3e5aa8c3b3adbca1903aa6b00f041557296bd452 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:00:45 -0700 Subject: [PATCH 54/70] topology: mark replica quarantined before recording the pair (#11983) * topology: serialize quarantine records with their layout marks A quarantine publishes a (node, vid) record and marks the replica read-only at the layout level; release and heartbeat flag updates must observe that pair as one state. Store-then-mark let a concurrent release delete the record before the mark landed, stranding the surviving replica read-only; mark-then-store let a heartbeat clear the mark before the record was visible, returning a conflicting replica to the writable list. quarantineLock now covers every record<->mark pair: flagDuplicateVolumeId's quarantine, releaseResolvedQuarantine's undo, the isQuarantined-gated marks on registration, and the quarantine-folded read-only flag writes on heartbeat paths. Remote server pushes stay outside the lock; a push that lands after the record was released is undone, and a failed release push re-adds the record for a later retry. The release test now waits for in-flight async flags to settle instead of asserting on a single read. * topology: fold quarantine into RegisterVolume's flag rebuild RegisterVolume rewrites every replica's location read-only flag from what its node last reported, and reported flags deliberately exclude quarantine. Any registration of a shared vid -- a reconnected replica, a new copy, the self-heal and delta-heartbeat re-register paths -- therefore cleared the quarantine mark on every replica in the location, while the gate afterwards only re-marked the registering node. Two call sites had no gate at all. RegisterVolume now takes an isQuarantined predicate and evaluates it under accessLock for every replica it touches, so the mark is never dropped in the first place: a release unmarks after the rebuild, and a quarantine mark re-applies on the next rebuild, with no window between. * topology: keep quarantine when a vacuum commit re-adds a replica SetVolumeAvailable re-adds the replica into the location list; a re-added slot starts with a cleared read-only flag, so a quarantined replica came back flag-free and the next ensureCorrectWritables could return a conflicting volume to the writable list. Pass the quarantine state in and set the flag back under the same accessLock, with the caller holding quarantineLock so release cannot interleave between the check and the mark. * topology: merge quarantine records instead of overwriting them enforceQuarantine wrote back the state copy it loaded before its remote push, so a concurrent writer's enforced=true was lost: the server stayed read-only while the record said unenforced, and a later release skipped the writable push it owed. The write-back now reloads the record under quarantineLock and merges -- enforced wins from either side, reported is taken from the newer pre-enforcement observation only while no push has landed yet. All three compensating writable pushes had the same hole in reverse: a failed un-quarantine left the server read-only with no record to retry from, and release's restore blindly overwrote any record a fresh flag pass had created in between. They now go through restoreQuarantineRecord, which adds the record only when absent and otherwise sets enforced on the current one, preserving its reported flag. * topology: hold quarantineLock over registration's quarantine check-and-mark RegisterVolume reads the quarantine record and marks the replica read-only under accessLock, but the pair ran without quarantineLock. A release could push writable, watch the record get re-read, and then delete it: the replica stayed read-only with no record left to clear, and unchanged writable heartbeats skip SetReplicaReadOnlyFlag so the mark never repaired. Holding quarantineLock around the check-and-mark in all three production callers serializes it against the release clear-and-delete. * topology: serialize remote quarantine pushes behind a record recheck releaseResolvedQuarantine deletes the record under quarantineLock and sends the writable request after unlock. A conflict flagged while that request is in flight creates a fresh record and marks the replica read-only again, and the older writable request can still land last, leaving the server writable under an active quarantine. The reverse flap strands a stale read-only mark the same way. Route every QuarantineVolumeFn call through pushQuarantine, which holds quarantinePushLock across the remote call and re-checks the record under quarantineLock right before sending: a read-only push requires the pair to exist, a writable push requires it to be gone. A skipped push means the decision already moved on, so the final server state always follows the newest record. * topology: serialize quarantine pushes per replica, not globally A single quarantinePushLock held across the remote mark call let one slow volume server (up to the 10s client timeout) delay pushes for unrelated replicas. Keying the lock by (node, vid) keeps pushes to one replica ordered in record-decision order while pushes to others run in parallel. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: restore the layout read-only mark with a released record A failed writable push re-armed the quarantine record but left the replica marked writable in the layout, so PickForWrite could route to a replica the server still holds read-only. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: wait out async duplicate checks in the release-retry test TestFailedReleasePushRestoresQuarantineMark asserts layout marks around a release pass, but the two RegisterVolumeLayout calls each start a background flagDuplicateVolumeId that can still be mid-flight. A late mark could land after the final release and fail the writable assertion even though the code behaved correctly. Track the async passes in a WaitGroup and drain them before the explicit flag call so the rest of the test is deterministic. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * topology: keep quarantine when a replica's layout key is unreadable * topology: keep a failed writable push's restore ordered under the push lock * topology: retry the quarantine release on every duplicate check * ci: isolate Java and S3 test servers on persistent runners * ci: drop the duplicated -ip flags in the lifecycle test server The bind flags were already on the command; the isolation pass added a second pair. * topology: retry a failed quarantine writable push on the next report * topology: retry pending quarantine releases off the leader pulse A failed writable push re-arms the quarantine record with a retry that only enforceQuarantine fires, and that runs only for volumes a heartbeat carries. Once the server has reported the replica read-only it drops out of the delta reports, and restoreQuarantineRecord keeps the stored flag matching the server's so the digests agree and no full list is requested - the pending release then never retries and the idle volume stays read-only forever. Run the same release pass from the leader's pulse loop so a pending retry no longer waits on a report. * topology: exercise the pulse-driven quarantine release retry --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/java_integration_tests.yml | 1 + .github/workflows/s3-go-tests.yml | 8 +- test/s3/versioning/Makefile | 7 +- ...master_grpc_server_changed_volumes_test.go | 2 +- weed/topology/duplicate_volume_id_test.go | 142 +++++++- weed/topology/maintenance_mode_test.go | 2 +- weed/topology/topology.go | 310 ++++++++++++++++-- weed/topology/topology_event_handling.go | 1 + weed/topology/topology_test.go | 2 +- weed/topology/topology_vacuum.go | 4 +- weed/topology/volume_growth_test.go | 2 +- weed/topology/volume_layout.go | 47 +-- .../volume_layout_capacity_recovery_test.go | 2 +- weed/topology/volume_layout_pick_test.go | 2 +- 14 files changed, 464 insertions(+), 68 deletions(-) diff --git a/.github/workflows/java_integration_tests.yml b/.github/workflows/java_integration_tests.yml index 363c424768..cdb61b3faf 100644 --- a/.github/workflows/java_integration_tests.yml +++ b/.github/workflows/java_integration_tests.yml @@ -60,6 +60,7 @@ jobs: # Start SeaweedFS with optimized settings for CI # Include S3 API for s3copier integration tests weed server -dir="$WEED_DATA_DIR" \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -master.raftHashicorp \ -master.electionTimeout=1s \ -master.volumeSizeLimitMB=100 \ diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 6f5de41974..7660ce1d71 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -54,7 +54,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + df -h . /tmp echo "=== Starting Tests ===" # Run tests with automatic server management @@ -220,7 +220,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + df -h . /tmp echo "=== Starting Tests ===" # Run tests with automatic server management @@ -358,7 +358,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + df -h . /tmp echo "=== Starting Tests ===" make test-with-server @@ -417,7 +417,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + df -h . /tmp echo "=== Starting Tests ===" # Run tests with automatic server management diff --git a/test/s3/versioning/Makefile b/test/s3/versioning/Makefile index f41cfe3905..26ec27704f 100644 --- a/test/s3/versioning/Makefile +++ b/test/s3/versioning/Makefile @@ -94,6 +94,7 @@ start-server: check-deps @echo "🔍 DEBUG: Launching SeaweedFS server in background..." @echo "🔍 DEBUG: Command: $(WEED_BINARY) mini -dir=./test-volume-data -s3.port=$(S3_PORT) -s3.config=../../../docker/compose/s3.json" @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -dir=./test-volume-data \ -s3.port=$(S3_PORT) \ -s3.config=../../../docker/compose/s3.json \ @@ -229,13 +230,13 @@ test-with-server: start-server test-versioning-with-configs: check-deps @echo "Testing with different S3 configurations..." @echo "Testing with delete bucket not empty allowed..." - @$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid + @$(WEED_BINARY) mini -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=true > weed-test-config1.log 2>&1 & echo $$! > weed-config1.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config1.pid ]; then kill -TERM $$(cat weed-config1.pid) 2>/dev/null || true; rm -f weed-config1.pid; fi @sleep 2 @echo "Testing with delete bucket not empty disabled..." - @$(WEED_BINARY) mini -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid + @$(WEED_BINARY) mini -ip=127.0.0.1 -ip.bind=127.0.0.1 -s3.port=$(S3_PORT) -s3.allowDeleteBucketNotEmpty=false > weed-test-config2.log 2>&1 & echo $$! > weed-config2.pid @sleep 5 @go test -v -timeout=5m -run "TestVersioningBasicWorkflow" . || true @if [ -f weed-config2.pid ]; then kill -TERM $$(cat weed-config2.pid) 2>/dev/null || true; rm -f weed-config2.pid; fi @@ -279,6 +280,7 @@ debug-server: @$(MAKE) stop-server @mkdir -p ./test-volume-data @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -debug \ -s3.port=$(S3_PORT) \ -s3.allowDeleteBucketNotEmpty=true \ @@ -323,6 +325,7 @@ health-check: start-server-simple: check-deps @echo "Starting SeaweedFS server (simple mode)..." @$(WEED_BINARY) mini \ + -ip=127.0.0.1 -ip.bind=127.0.0.1 \ -dir=$(SERVER_DIR) \ -s3 \ -s3.port=$(S3_PORT) \ diff --git a/weed/server/master_grpc_server_changed_volumes_test.go b/weed/server/master_grpc_server_changed_volumes_test.go index 5436418fb1..0ed15aa132 100644 --- a/weed/server/master_grpc_server_changed_volumes_test.go +++ b/weed/server/master_grpc_server_changed_volumes_test.go @@ -220,7 +220,7 @@ func TestRegistrationRacingCollectionDeleteDoesNotLeak(t *testing.T) { topo.DeleteCollection("c") // the interleaved registration must refuse the dropped layout - if stale.RegisterVolume(&vi, dn) { + if stale.RegisterVolume(&vi, dn, nil) { t.Fatal("registered into a layout dropped with its collection") } diff --git a/weed/topology/duplicate_volume_id_test.go b/weed/topology/duplicate_volume_id_test.go index 42cb6fd97a..ec0ed4aeda 100644 --- a/weed/topology/duplicate_volume_id_test.go +++ b/weed/topology/duplicate_volume_id_test.go @@ -1,7 +1,12 @@ package topology import ( + "errors" + "sync/atomic" "testing" + "time" + + "github.com/stretchr/testify/require" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" "github.com/seaweedfs/seaweedfs/weed/storage" @@ -104,8 +109,102 @@ func TestQuarantineReleasedWhenConflictResolves(t *testing.T) { topo.UnRegisterVolumeLayout(v2, dn2) + // Registration flags duplicates asynchronously, so a flag that was still + // in flight when the conflict resolved releases the surviving copy only + // once it settles. Wait for that release instead of reading once. + deadline := time.Now().Add(5 * time.Second) + for !isWritable(t, topo, v1) { + if time.Now().After(deadline) { + t.Fatal("the surviving copy must be released once the conflict is gone") + } + time.Sleep(10 * time.Millisecond) + } +} + +func TestFailedReleasePushRestoresQuarantineMark(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + + var pushFails atomic.Bool + topo.QuarantineVolumeFn = func(dn *DataNode, vid needle.VolumeId, readOnly bool) error { + if !readOnly && pushFails.Load() { + return errors.New("volume server unreachable") + } + return nil + } + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v2, dn2) + // Registrations flag duplicates asynchronously; let them settle so no + // late mark can land after the release assertions below. + topo.duplicateFlagChecks.Wait() + topo.flagDuplicateVolumeId(v2, dn2) + + // the conflict resolves while the surviving copy is unreachable: the + // writable push fails, so the record and the layout read-only mark must + // both come back instead of advertising a writable replica the server + // still holds read-only + pushFails.Store(true) + topo.UnRegisterVolumeLayout(v2, dn2) + + if !topo.isQuarantined(dn1, v1.Id) { + t.Fatal("a failed writable push must keep the quarantine record for the retry") + } + if isWritable(t, topo, v1) { + t.Fatal("the layout must not advertise a replica the server still holds read-only") + } + + pushFails.Store(false) + // An idle replica drops out of the delta reports once its stored flag + // matches the server's, so the release pass retries off the leader pulse + // rather than waiting for the volume to be reported again. + topo.retryPendingQuarantineReleases() + require.Eventually(t, func() bool { + return !topo.isQuarantined(dn1, v1.Id) && isWritable(t, topo, v1) + }, 5*time.Second, time.Millisecond, "the pending release must retry without another volume report") +} + +func TestQuarantineKeptWhenReplicaUnreadable(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v2, dn2) + topo.duplicateFlagChecks.Wait() + topo.flagDuplicateVolumeId(v2, dn2) + + // the surviving copy drops out of its node's disk map while the lookup + // index still holds it; its layout key is then unknown, so the release + // pass must keep the mark rather than assume the conflict resolved + dn1.UpdateVolumes(nil) + topo.UnRegisterVolumeLayout(v2, dn2) + + if !topo.isQuarantined(dn1, v1.Id) { + t.Fatal("an unreadable replica must not lose its quarantine record") + } + if isWritable(t, topo, v1) { + t.Fatal("a replica whose conflict cannot be ruled out must stay read-only") + } + + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + topo.RegisterVolumeLayout(v1, dn1) + topo.duplicateFlagChecks.Wait() if !isWritable(t, topo, v1) { - t.Fatal("the surviving copy must be released once the conflict is gone") + t.Fatal("once readable again and sole, the copy must be released to writable") } } @@ -134,6 +233,47 @@ func TestQuarantineSurvivesResync(t *testing.T) { } } +func TestQuarantineSurvivesReplicaRejoin(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("001") + + dn1 := NewDataNode("server1") + dn1.Ip = "10.0.0.1" + dn1.Port = 8080 + dn2 := NewDataNode("server2") + dn2.Ip = "10.0.0.2" + dn2.Port = 8080 + dn3 := NewDataNode("server3") + dn3.Ip = "10.0.0.3" + dn3.Port = 8080 + + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v1}) + dn3.UpdateVolumes([]storage.VolumeInfo{v2}) + topo.RegisterVolumeLayout(v1, dn1) + topo.RegisterVolumeLayout(v1, dn2) + topo.RegisterVolumeLayout(v2, dn3) + topo.flagDuplicateVolumeId(v2, dn3) + + // a replica dropped from the lookup index re-registers through the + // self-heal path; rebuilding location flags from reported state must + // not lift the quarantine on any copy + vl := topo.FindVolumeLayout(v1.Collection, v1.ReplicaPlacement, v1.Ttl, types.ToDiskType(v1.DiskType)) + vl.SetVolumeUnavailable(dn1, v1.Id) + topo.SyncDataNodeRegistration([]*master_pb.VolumeInformationMessage{{ + Id: 7, Collection: "a", ReplicaPlacement: uint32(rp.Byte()), Version: 3, + }}, dn1) + + if isWritable(t, topo, v1) { + t.Fatal("a re-registering replica must not clear the quarantine on the surviving copies") + } + if isWritable(t, topo, v2) { + t.Fatal("the conflicting copy must stay quarantined too") + } +} + func TestReplicationChangeIsNotDuplicate(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) rpOld, _ := super_block.NewReplicaPlacementFromString("000") diff --git a/weed/topology/maintenance_mode_test.go b/weed/topology/maintenance_mode_test.go index 27c3f580ab..596ece558f 100644 --- a/weed/topology/maintenance_mode_test.go +++ b/weed/topology/maintenance_mode_test.go @@ -231,7 +231,7 @@ func TestMaintenanceModeHoldsThroughVacuumAndMarkWritable(t *testing.T) { vl.DrainAndRemoveFromWritable(1) topo.SetDataNodeMaintenanceMode(dn1, true) for _, dn := range []*DataNode{dn1, dn2} { - if vl.SetVolumeAvailable(dn, 1, false, false) { + if vl.SetVolumeAvailable(dn, 1, false, false, false) { t.Errorf("vacuum commit on %s made the volume writable with a replica in maintenance", dn.Id()) } } diff --git a/weed/topology/topology.go b/weed/topology/topology.go index cee0018584..07f1ffa765 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -106,6 +106,21 @@ type Topology struct { // vid collided across layouts. Kept separate from the reported flag so // later heartbeats from the same server cannot quietly undo it. quarantinedVolumeIds sync.Map + // quarantineLock serializes each quarantine record update with the layout + // mark it justifies: a release pass and a heartbeat read-only flag update + // each observe record and mark as one state, never a mark whose record + // was already deleted or a cleared mark whose record is not yet visible. + quarantineLock sync.Mutex + // quarantinePushLocks serialize the remote mark/unmark calls per replica + // so a push decided under an older record state cannot land on the + // volume server after a newer one. Keyed by replica so a slow server + // does not stall pushes to the others. + quarantinePushLocks map[quarantinedVolume]*quarantinePushLock + quarantinePushLocksMu sync.Mutex + + // duplicateFlagChecks tracks in-flight flagDuplicateVolumeId passes so + // tests can wait for the async checks a registration started. + duplicateFlagChecks sync.WaitGroup // QuarantineVolumeFn pushes the quarantine onto the volume server itself // (VolumeMarkReadonly/VolumeMarkWritable). nil keeps quarantine at the @@ -121,6 +136,12 @@ type quarantinedVolume struct { type quarantinedVolumeState struct { reported bool enforced bool + retry bool +} + +type quarantinePushLock struct { + sync.Mutex + refs int } func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { @@ -128,6 +149,102 @@ func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { return ok } +// pushQuarantine sends the remote mark only while it still matches the +// record: a read-only push requires the pair to exist, a writable push +// requires it to be gone. The bool reports whether the remote call was +// actually made, so callers can tell a skipped stale push from a failed one. +func (t *Topology) pushQuarantine(dn *DataNode, vid needle.VolumeId, readOnly bool) (bool, error) { + unlock := t.lockQuarantinePush(quarantinedVolume{dn.Id(), vid}) + defer unlock() + return t.sendQuarantine(dn, vid, readOnly) +} + +// sendQuarantine runs the record check and remote mark; the caller holds +// the replica's push lock. +func (t *Topology) sendQuarantine(dn *DataNode, vid needle.VolumeId, readOnly bool) (bool, error) { + t.quarantineLock.Lock() + _, exists := t.quarantinedVolumeIds.Load(quarantinedVolume{dn.Id(), vid}) + t.quarantineLock.Unlock() + if exists != readOnly { + return false, nil + } + return true, t.QuarantineVolumeFn(dn, vid, readOnly) +} + +// releaseQuarantineWritable pushes the writable mark and re-arms the record +// on failure, all under the replica's push lock so a queued stale push cannot +// slip between the failure and the restore and undo the re-armed record. +func (t *Topology) releaseQuarantineWritable(dn *DataNode, vid needle.VolumeId) { + unlock := t.lockQuarantinePush(quarantinedVolume{dn.Id(), vid}) + defer unlock() + if _, err := t.sendQuarantine(dn, vid, false); err != nil { + t.restoreQuarantineRecord(dn, vid) + } +} + +// lockQuarantinePush returns the release for q's push lock, creating the lock +// on first use and dropping it once the last holder lets go. +func (t *Topology) lockQuarantinePush(q quarantinedVolume) func() { + t.quarantinePushLocksMu.Lock() + if t.quarantinePushLocks == nil { + t.quarantinePushLocks = make(map[quarantinedVolume]*quarantinePushLock) + } + l, ok := t.quarantinePushLocks[q] + if !ok { + l = &quarantinePushLock{} + t.quarantinePushLocks[q] = l + } + l.refs++ + t.quarantinePushLocksMu.Unlock() + l.Lock() + return func() { + t.quarantinePushLocksMu.Lock() + l.refs-- + if l.refs == 0 { + delete(t.quarantinePushLocks, q) + } + t.quarantinePushLocksMu.Unlock() + l.Unlock() + } +} + +// takeQuarantineRetryLocked disarms a pending release retry and reports +// whether one was armed. The caller holds quarantineLock. +func (t *Topology) takeQuarantineRetryLocked(q quarantinedVolume) bool { + v, ok := t.quarantinedVolumeIds.Load(q) + if !ok { + return false + } + state := v.(quarantinedVolumeState) + if !state.retry { + return false + } + state.retry = false + t.quarantinedVolumeIds.Store(q, state) + return true +} + +// retryPendingQuarantineReleases re-runs the release pass for every record a +// failed writable push re-armed. The retry in enforceQuarantine only fires +// while a heartbeat keeps carrying the volume, and an idle replica drops out +// of the delta reports once its stored flag matches the server's, so pending +// retries also run off the leader pulse. +func (t *Topology) retryPendingQuarantineReleases() { + vids := make(map[needle.VolumeId]struct{}) + t.quarantineLock.Lock() + t.quarantinedVolumeIds.Range(func(key, _ any) bool { + q := key.(quarantinedVolume) + if t.takeQuarantineRetryLocked(q) { + vids[q.vid] = struct{}{} + } + return true + }) + t.quarantineLock.Unlock() + for vid := range vids { + go t.releaseResolvedQuarantine(vid) + } +} + // enforceQuarantine keeps the latest reported read-only flag for a quarantined // copy so a later release restores the server's true state. The stored report // keeps the server's flag: quarantine lives at the layout level, and folding @@ -135,19 +252,55 @@ func (t *Topology) isQuarantined(dn *DataNode, vid needle.VolumeId) bool { // disagree with the server's and force a full volume list on every heartbeat. func (t *Topology) enforceQuarantine(dn *DataNode, vi storage.VolumeInfo) storage.VolumeInfo { q := quarantinedVolume{dn.Id(), vi.Id} - if v, ok := t.quarantinedVolumeIds.Load(q); ok { - state := v.(quarantinedVolumeState) - if !state.enforced { - state.reported = vi.ReadOnly - } - if t.QuarantineVolumeFn != nil && (!state.enforced || !vi.ReadOnly) { - // The mark never landed, or the server lost it (restart, manual - // unquarantine); push it while the conflict stands. - if t.QuarantineVolumeFn(dn, vi.Id, true) == nil { - state.enforced = true - } - } + t.quarantineLock.Lock() + v, ok := t.quarantinedVolumeIds.Load(q) + if !ok { + t.quarantineLock.Unlock() + return vi + } + if t.takeQuarantineRetryLocked(q) { + // A failed writable push re-armed this record; hand the release pass + // one retry per report. Clearing it first keeps a surviving conflict + // on the normal enforce path afterwards. + t.quarantineLock.Unlock() + go t.releaseResolvedQuarantine(vi.Id) + return vi + } + state := v.(quarantinedVolumeState) + if !state.enforced { + state.reported = vi.ReadOnly + } + if t.QuarantineVolumeFn == nil || (state.enforced && vi.ReadOnly) { t.quarantinedVolumeIds.Store(q, state) + t.quarantineLock.Unlock() + return vi + } + t.quarantineLock.Unlock() + + // The mark never landed on the server, or the server lost it (restart, + // manual unquarantine); push it while the conflict stands. The call is + // remote, so it runs outside the lock. + pushed, err := t.pushQuarantine(dn, vi.Id, true) + if pushed && err == nil { + state.enforced = true + } + t.quarantineLock.Lock() + cur, still := t.quarantinedVolumeIds.Load(q) + if still { + latest := cur.(quarantinedVolumeState) + if !latest.enforced { + latest.reported = state.reported + } + latest.enforced = latest.enforced || state.enforced + t.quarantinedVolumeIds.Store(q, latest) + t.quarantineLock.Unlock() + return vi + } + t.quarantineLock.Unlock() + if state.enforced && !state.reported { + // The conflict resolved while the push was in flight; leave the + // server consistent with the released layout. + t.releaseQuarantineWritable(dn, vi.Id) } return vi } @@ -173,6 +326,7 @@ func NewTopology(id string, seq sequence.Sequencer, volumeSizeLimit uint64, puls t.Configuration = &Configuration{} t.dataNodeIndex = make(map[string]*DataNode) + t.quarantinePushLocks = make(map[quarantinedVolume]*quarantinePushLock) return t } @@ -678,18 +832,31 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { diskType := types.ToDiskType(v.DiskType) for { vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) - if vl.RegisterVolume(&v, dn) { - if t.isQuarantined(dn, v.Id) { - vl.SetVolumeQuarantined(dn, v.Id) - } + // RegisterVolume reads the quarantine record and marks under + // accessLock; holding quarantineLock keeps that pair serialized with + // releaseResolvedQuarantine's clear-and-delete. + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&v, dn, func(n *DataNode) bool { + return t.isQuarantined(n, v.Id) + }) + t.quarantineLock.Unlock() + if registered { vl.EnsureCorrectWritables(&v) - go t.flagDuplicateVolumeId(v, dn) + t.flagDuplicateVolumeIdAsync(v, dn) return } // Dropped with its collection; the next lookup creates a fresh one. } } +func (t *Topology) flagDuplicateVolumeIdAsync(v storage.VolumeInfo, dn *DataNode) { + t.duplicateFlagChecks.Add(1) + go func() { + defer t.duplicateFlagChecks.Done() + t.flagDuplicateVolumeId(v, dn) + }() +} + // flagDuplicateVolumeId reports and quarantines v.Id when another data node // already serves it under a different layout key. It runs after registration // so a legitimate same-node move (disk type or collection change) settles @@ -697,13 +864,14 @@ func (t *Topology) RegisterVolumeLayout(v storage.VolumeInfo, dn *DataNode) { // quarantined: either could be the impostor, and arrival order must not // decide which copy stays writable. func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { + // Every pass ends with a release: a conflict could resolve between the + // scan below and the marks, and a check finding no hits is also the retry + // a kept record waits for once its replica reports a readable layout. + defer t.releaseResolvedQuarantine(v.Id) hits := t.findDuplicateVolumeLayouts(v, dn) if len(hits) == 0 { return } - // A conflict could resolve between the scan above and the marks below; - // re-run the release so a stale quarantine is cleared instead of kept. - defer t.releaseResolvedQuarantine(v.Id) stats.MasterDuplicateVolumeIds.WithLabelValues(v.Collection, v.Id.String()).Inc() var otherKeys []string for _, h := range hits { @@ -716,16 +884,33 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { if vi, err := n.GetVolumesById(v.Id); err == nil { reported = vi.ReadOnly } + t.quarantineLock.Lock() t.quarantinedVolumeIds.LoadOrStore(q, quarantinedVolumeState{reported: reported}) if vl != nil { vl.SetVolumeQuarantined(n, v.Id) } - if t.QuarantineVolumeFn != nil && t.QuarantineVolumeFn(n, v.Id, true) == nil { - if val, ok := t.quarantinedVolumeIds.Load(q); ok { - state := val.(quarantinedVolumeState) - state.enforced = true - t.quarantinedVolumeIds.Store(q, state) - } + t.quarantineLock.Unlock() + if t.QuarantineVolumeFn == nil { + return + } + pushed, err := t.pushQuarantine(n, v.Id, true) + if err != nil { + return + } + t.quarantineLock.Lock() + val, ok := t.quarantinedVolumeIds.Load(q) + if ok { + state := val.(quarantinedVolumeState) + state.enforced = state.enforced || pushed + t.quarantinedVolumeIds.Store(q, state) + t.quarantineLock.Unlock() + return + } + t.quarantineLock.Unlock() + if pushed && !reported { + // The conflict resolved while the push was in flight; leave the + // server consistent with the released layout. + t.releaseQuarantineWritable(n, v.Id) } } quarantine(dn, t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType))) @@ -738,7 +923,14 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { // releaseResolvedQuarantine lifts read-only marks once the conflicting copy // is gone, so a finished move does not leave the surviving copy quarantined. +// The writable push is a remote call, so the locked pass only applies the +// layout marks and records which replicas still need it; the pushes run +// after the lock is dropped, a skipped push means a newer record re-armed +// the pair so its quarantine stands, and a failed push re-adds the record +// so a later release pass retries it. func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { + var writablePushes []*DataNode + t.quarantineLock.Lock() t.quarantinedVolumeIds.Range(func(key, val any) bool { q := key.(quarantinedVolume) if q.vid != vid { @@ -749,6 +941,12 @@ func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { t.quarantinedVolumeIds.Delete(key) return true } + if vi.ReplicaPlacement == nil { + // The replica is in the lookup index but unreadable on its node, + // so its layout key is unknown and the conflict cannot be ruled + // out; keep the mark for a later pass. + return true + } if hits := t.findDuplicateVolumeLayouts(vi, dn); len(hits) > 0 { return true } @@ -759,16 +957,38 @@ func (t *Topology) releaseResolvedQuarantine(vid needle.VolumeId) { } else { vl.SetVolumeWritable(dn, vid) if state.enforced && t.QuarantineVolumeFn != nil { - if err := t.QuarantineVolumeFn(dn, vid, false); err != nil { - // The server still rejects writes; keep the mark so a - // later release pass retries the writable push. - return true - } + writablePushes = append(writablePushes, dn) } } t.quarantinedVolumeIds.Delete(key) return true }) + t.quarantineLock.Unlock() + for _, dn := range writablePushes { + t.releaseQuarantineWritable(dn, vid) + } +} + +// restoreQuarantineRecord re-adds a released record after a failed writable +// push so the next volume report retries the release. An existing record keeps its +// reported flag; the server is known read-only, so enforced and retry are set. +// A record restored from scratch also gets its layout read-only mark back: +// the release pass cleared it with the record, and leaving it cleared would +// advertise a writable replica the server still holds read-only. +func (t *Topology) restoreQuarantineRecord(dn *DataNode, vid needle.VolumeId) { + t.quarantineLock.Lock() + defer t.quarantineLock.Unlock() + key := quarantinedVolume{dn.Id(), vid} + if cur, ok := t.quarantinedVolumeIds.LoadOrStore(key, quarantinedVolumeState{enforced: true, retry: true}); ok { + state := cur.(quarantinedVolumeState) + state.enforced = true + state.retry = true + t.quarantinedVolumeIds.Store(key, state) + return + } + if _, vl, _ := t.findQuarantinedVolume(dn.Id(), vid); vl != nil { + vl.SetVolumeReadOnly(dn, vid) + } } func (t *Topology) findQuarantinedVolume(dnId NodeId, vid needle.VolumeId) (*DataNode, *VolumeLayout, storage.VolumeInfo) { @@ -983,7 +1203,9 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati if v.ReplicaPlacement == nil { continue } + t.quarantineLock.Lock() t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)).SetReplicaReadOnlyFlag(dn, v.Id, v.ReadOnly || t.isQuarantined(dn, v.Id)) + t.quarantineLock.Unlock() } // Update effective sizes for all reported volumes (decay pending estimates). // If decay brings a volume eagerly removed by RecordAssign back under the @@ -1002,12 +1224,20 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati // Without this, the volume stays visible in volume.list/admin UI yet // LookupVolume returns "volume id not found". if !vl.HasDataNode(v.Id, dn) { - for !vl.RegisterVolume(&v, dn) { + for { + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&v, dn, func(n *DataNode) bool { + return t.isQuarantined(n, v.Id) + }) + t.quarantineLock.Unlock() + if registered { + break + } // Dropped with its collection; the next lookup creates a fresh // one, which the calls below must use too. vl = t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, diskType) } - go t.flagDuplicateVolumeId(v, dn) + t.flagDuplicateVolumeIdAsync(v, dn) // Volumes new to the disk map were registered above, so reaching // here means only the lookup index had lost it. Clients were told // it went when the node dropped out, so the repair has to tell them @@ -1108,18 +1338,28 @@ func (t *Topology) ApplyVolumeChanges(changed []*master_pb.VolumeInformationMess // volume only that index had lost is an arrival as far as clients are // concerned: they were told it went when the node dropped out. becameServable := !vl.HasDataNode(vi.Id, dn) - for becameServable && !vl.RegisterVolume(&vi, dn) { + for becameServable { + t.quarantineLock.Lock() + registered := vl.RegisterVolume(&vi, dn, func(n *DataNode) bool { + return t.isQuarantined(n, vi.Id) + }) + t.quarantineLock.Unlock() + if registered { + break + } // Dropped with its collection; the next lookup creates a fresh one. vl = t.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, vi.Ttl, types.ToDiskType(vi.DiskType)) } if becameServable { - go t.flagDuplicateVolumeId(vi, dn) + t.flagDuplicateVolumeIdAsync(vi, dn) } if isNew || becameServable || tierTransition || isChanged { newVolumes = append(newVolumes, vi) } if isChanged { + t.quarantineLock.Lock() vl.SetReplicaReadOnlyFlag(dn, vi.Id, vi.ReadOnly || t.isQuarantined(dn, vi.Id)) + t.quarantineLock.Unlock() } vl.UpdateOversizedState(&vi, dn) if vl.UpdateVolumeSize(vi.Id, vi.Size, vi.CompactRevision, true) { diff --git a/weed/topology/topology_event_handling.go b/weed/topology/topology_event_handling.go index 9dcd6f93e3..1793ff0e93 100644 --- a/weed/topology/topology_event_handling.go +++ b/weed/topology/topology_event_handling.go @@ -23,6 +23,7 @@ func (t *Topology) StartRefreshWritableVolumes(grpcDialOption grpc.DialOption, g freshThreshHold := time.Now().Unix() - 3*t.pulse //3 times of sleep interval t.CollectDeadNodeAndFullVolumes(freshThreshHold, t.volumeSizeLimit, growThreshold) t.DecayQuietVolumeSizes() + t.retryPendingQuarantineReleases() } time.Sleep(time.Duration(float32(t.pulse*1e3)*(1+rand.Float32())) * time.Millisecond) } diff --git a/weed/topology/topology_test.go b/weed/topology/topology_test.go index 3a89ce14f5..e761ede3c5 100644 --- a/weed/topology/topology_test.go +++ b/weed/topology/topology_test.go @@ -851,7 +851,7 @@ func TestSetVolumeAvailableRepairsMissingVolume(t *testing.T) { } // The vacuum commit re-marks the volume available; it must re-register it. - vl.SetVolumeAvailable(dn, vid, false, false) + vl.SetVolumeAvailable(dn, vid, false, false, false) if got := topo.Lookup("", vid); len(got) != 1 { t.Fatalf("after SetVolumeAvailable: lookup %d got %v, want 1 location", vid, got) } diff --git a/weed/topology/topology_vacuum.go b/weed/topology/topology_vacuum.go index 0cc779d0ea..ea6de330f4 100644 --- a/weed/topology/topology_vacuum.go +++ b/weed/topology/topology_vacuum.go @@ -226,7 +226,9 @@ func (t *Topology) batchVacuumVolumeCommit(grpcDialOption grpc.DialOption, vl *V vl.accessLock.Unlock() for _, dn := range vacuumLocationList.list { - vl.SetVolumeAvailable(dn, vid, isReadOnly, isFullCapacity) + t.quarantineLock.Lock() + vl.SetVolumeAvailable(dn, vid, isReadOnly, isFullCapacity, t.isQuarantined(dn, vid)) + t.quarantineLock.Unlock() } } return isCommitSuccess diff --git a/weed/topology/volume_growth_test.go b/weed/topology/volume_growth_test.go index 11e3ba0829..da9cf5affd 100644 --- a/weed/topology/volume_growth_test.go +++ b/weed/topology/volume_growth_test.go @@ -116,7 +116,7 @@ func setup(topologyLayout string) *Topology { } if vi.ReplicaPlacement != nil { vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType) - vl.RegisterVolume(&vi, server) + vl.RegisterVolume(&vi, server, nil) vl.setVolumeWritable(vi.Id) } server.AddOrUpdateVolume(vi) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index b0a48e750b..c74fefccbc 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -132,7 +132,11 @@ func (vl *VolumeLayout) initSizeTracking(vid needle.VolumeId, size uint64, compa // RegisterVolume records a volume location. It refuses a layout dropped with // its collection — the caller must re-resolve, or the bits it sets would leak. -func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) bool { +// Each replica's read-only flag is rebuilt from what its node last reported; +// isQuarantined folds the master's force-read-only marks into that rebuild, +// since reported flags deliberately exclude them. nil restores reported state +// verbatim. +func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode, isQuarantined func(*DataNode) bool) bool { vl.accessLock.Lock() defer vl.accessLock.Unlock() @@ -140,27 +144,27 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) bool return false } - moveLookupOwnership(v.Id, vl.getOrCreateLocationList(v.Id).Set(dn), dn) + location := vl.getOrCreateLocationList(v.Id) + moveLookupOwnership(v.Id, location.Set(dn), dn) if !v.ReadOnly { vl.initSizeTracking(v.Id, v.Size, v.CompactRevision) } + if isQuarantined != nil && isQuarantined(dn) { + location.SetReadOnly(dn, true) + vl.removeFromWritable(v.Id) + } // glog.V(4).Infof("volume %d added to %s len %d copy %d", v.Id, dn.Id(), vl.vid2location[v.Id].Length(), v.ReplicaPlacement.GetCopyCount()) - location := vl.vid2location[v.Id] - for _, dn := range location.list { - if vInfo, err := dn.GetVolumesById(v.Id); err == nil { - if vInfo.ReadOnly { - glog.V(1).Infof("vid %d removed from writable", v.Id) - vl.removeFromWritable(v.Id) - location.SetReadOnly(dn, true) - return true - } - location.SetReadOnly(dn, false) - } else { - glog.V(1).Infof("vid %d removed from writable", v.Id) - vl.removeFromWritable(v.Id) - location.SetReadOnly(dn, false) - return true + for _, n := range location.list { + marked := isQuarantined != nil && isQuarantined(n) + vInfo, err := n.GetVolumesById(v.Id) + if err == nil && !vInfo.ReadOnly && !marked { + location.SetReadOnly(n, false) + continue } + glog.V(1).Infof("vid %d removed from writable", v.Id) + vl.removeFromWritable(v.Id) + location.SetReadOnly(n, marked || (err == nil && vInfo.ReadOnly)) + return true } return true } @@ -1146,7 +1150,7 @@ func (vl *VolumeLayout) SetVolumeUnavailable(dn *DataNode, vid needle.VolumeId) } return false } -func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, isReadOnly, isFullCapacity bool) (becameWritable bool) { +func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, isReadOnly, isFullCapacity, quarantined bool) (becameWritable bool) { restoreActiveVolumeCount := false vl.accessLock.Lock() defer func() { @@ -1167,8 +1171,13 @@ func (vl *VolumeLayout) SetVolumeAvailable(dn *DataNode, vid needle.VolumeId, is // A disconnect during a long vacuum can drop the entry while the volume is // still on the node; re-create it instead of dereferencing a nil location, // so the commit also repairs the split. - moveLookupOwnership(vid, vl.getOrCreateLocationList(vid).Set(dn), dn) + location := vl.getOrCreateLocationList(vid) + moveLookupOwnership(vid, location.Set(dn), dn) + if quarantined { + location.SetReadOnly(dn, true) + return false + } if vInfo.ReadOnly || isReadOnly || isFullCapacity { return false } diff --git a/weed/topology/volume_layout_capacity_recovery_test.go b/weed/topology/volume_layout_capacity_recovery_test.go index 8edb9eead3..686c67e4ba 100644 --- a/weed/topology/volume_layout_capacity_recovery_test.go +++ b/weed/topology/volume_layout_capacity_recovery_test.go @@ -98,7 +98,7 @@ func TestSetVolumeAvailableRestoresActiveCountForCapacityFullVolume(t *testing.T dn := vl.vid2location[1].list[0] vl.accessLock.RUnlock() - if !vl.SetVolumeAvailable(dn, 1, false, false) { + if !vl.SetVolumeAvailable(dn, 1, false, false, false) { t.Fatalf("SetVolumeAvailable should report the volume became writable") } if got := topo.diskUsages.usages[types.HardDriveType].activeVolumeCount; got != initialActive { diff --git a/weed/topology/volume_layout_pick_test.go b/weed/topology/volume_layout_pick_test.go index 4fff615a65..aa8cc99c1f 100644 --- a/weed/topology/volume_layout_pick_test.go +++ b/weed/topology/volume_layout_pick_test.go @@ -61,7 +61,7 @@ func setupWithLimit(t testing.TB, topologyLayout string, volumeSizeLimit uint64) } if vi.ReplicaPlacement != nil { vl := topo.GetVolumeLayout(vi.Collection, vi.ReplicaPlacement, needle.EMPTY_TTL, types.HardDriveType) - vl.RegisterVolume(&vi, server) + vl.RegisterVolume(&vi, server, nil) vl.setVolumeWritable(vi.Id) } server.AddOrUpdateVolume(vi) From 8844d9823f561362025bf42a57c53ebe47bcd924 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:03:42 -0700 Subject: [PATCH 55/70] iam/oidc: wrap the jti replay cache in its own type (#12017) * iam/oidc: wrap the jti replay cache in its own type The replay-protection state lived as ten fields and nine methods spread across OIDCProvider. jtiReplayCache now owns the store, count, bound, sweep scheduling, and cleanup worker; the provider keeps a single jti field and delegates configure, consume, stop, adopt, and merge. * iam/oidc: lazily allocate the jti replay cache A zero-value OIDCProvider left jti nil, so Initialize, single-use validation, Shutdown, and the replay-state handoffs panicked on a nil cache. Allocate the cache through a sync.Once-guarded getter instead. Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/iam/oidc/jti_cache.go | 410 ++++++++++++++++++++++++++++ weed/iam/oidc/oidc_provider.go | 365 ++----------------------- weed/iam/oidc/oidc_provider_test.go | 44 +-- 3 files changed, 462 insertions(+), 357 deletions(-) create mode 100644 weed/iam/oidc/jti_cache.go diff --git a/weed/iam/oidc/jti_cache.go b/weed/iam/oidc/jti_cache.go new file mode 100644 index 0000000000..77dab0d044 --- /dev/null +++ b/weed/iam/oidc/jti_cache.go @@ -0,0 +1,410 @@ +package oidc + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/iam/providers" +) + +const ( + // clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519) + clockSkewTolerance = 5 * time.Minute + // defaultJTICacheSize is the default maximum number of JTI entries to cache + defaultJTICacheSize = 10000 + // defaultCleanupInterval is how often to scan for expired JTI entries + defaultCleanupInterval = 5 * time.Minute + // eagerCleanupMinInterval rate limits opportunistic sweeps near capacity + eagerCleanupMinInterval = 5 * time.Second + // tokenFingerprintPrefix marks replay-cache keys derived from the token's + // own fingerprint, used when the token carries no usable jti claim + tokenFingerprintPrefix = "fp:" +) + +// jtiEntry represents a cached JTI with metadata +type jtiEntry struct { + subject string + expiresAt time.Time + // uncounted marks an entry merged in past the capacity bound: it keeps + // rejecting replays but does not charge the count, so merged excess can + // neither reject new exchanges nor be double-counted when swept. + uncounted bool +} + +// jtiReplayCache wraps the consumed-token store that backs JTI replay +// protection. Each consumed token identifier is held until the token's exp +// plus clock skew, and a second presentation of the same identifier is +// rejected with ErrProviderTokenReplayed. +// +// The cache is bounded by maxSize and uses automatic TTL-based expiration: +// a background worker sweeps expired entries, an eager sweep runs when the +// cache nears capacity, and the consume path reclaims expired slots before +// rejecting a fresh token for capacity. +// +// Thread Safety: +// All methods are safe for concurrent use. The consume path uses atomic +// LoadOrStore to prevent TOCTOU races. +type jtiReplayCache struct { + name string // owning provider name, for log messages + + enabled atomic.Bool + maxSize atomic.Int64 + lastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups + + // storeMu guards the store/count pair: replay-state handoffs (adoptFrom, + // mergeFrom) repoint them under the write lock while the consume path + // holds a read lock, so a consume can never land in a store the handoff + // already drained and abandoned. + // It is always acquired before sweepMu, never while holding it. + storeMu sync.RWMutex + store *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys + count *atomic.Int64 // entries in store; shared with the store + + // mu guards the lifecycle of the cleanup worker: cleanupCancel and + // cleanupDone. The worker itself only touches store/count, so holding + // mu while waiting for it is safe. + mu sync.Mutex + cleanupCancel context.CancelFunc + cleanupDone chan struct{} // Closed when cleanup goroutine exits + + // sweepMu serializes cache sweeps so at most one scan runs at a time. + sweepMu sync.Mutex +} + +func newJTIReplayCache(name string) *jtiReplayCache { + return &jtiReplayCache{ + name: name, + store: &sync.Map{}, + count: &atomic.Int64{}, + } +} + +// configure applies the replay-protection settings: a previous worker is +// always stopped, whether the new configuration enables protection or +// disables it. The consumed-token store survives re-configuration: resetting +// it would let already-seen tokens be replayed against the refreshed provider. +func (c *jtiReplayCache) configure(enabled bool, maxSize int64) { + c.mu.Lock() + c.stopWorkerLocked() + if enabled { + if maxSize <= 0 { + maxSize = defaultJTICacheSize + } + // the bound must land before the flag: a consume that sees enabled + // with a zero bound would reject the first token it registers + c.maxSize.Store(maxSize) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + c.cleanupCancel = cancel + c.cleanupDone = done + c.enabled.Store(true) + c.mu.Unlock() + + go c.cleanupLoop(ctx, done, defaultCleanupInterval) + glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", c.name, maxSize) + } else { + c.enabled.Store(false) + c.mu.Unlock() + glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", c.name) + } +} + +// stopWorkerLocked cancels the current cleanup goroutine and waits briefly +// for it to exit. Caller must hold mu. +func (c *jtiReplayCache) stopWorkerLocked() { + if c.cleanupCancel == nil { + return + } + c.cleanupCancel() + select { + case <-c.cleanupDone: + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", c.name) + case <-time.After(2 * time.Second): + glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", c.name) + } + c.cleanupCancel = nil + c.cleanupDone = nil +} + +// consume marks a token as used and rejects a second presentation of it. +// jti is the token's jti claim (may be empty); tokenExpiresAt is its exp. +// A no-op while protection is disabled. +func (c *jtiReplayCache) consume(token, jti, subject string, tokenExpiresAt time.Time) error { + if !c.enabled.Load() { + return nil + } + + key := jti + if key == "" { + // Providers that issue no jti cannot be tracked by claim, so the + // signed input keys the entry: a repeated presentation of the same + // claims is a replay even when the signature bytes differ (e.g. + // ECDSA s/N-s malleability). + signingInput := token + if i := strings.LastIndexByte(token, '.'); i > 0 { + signingInput = token[:i] + } + sum := sha256.Sum256([]byte(signingInput)) + key = tokenFingerprintPrefix + hex.EncodeToString(sum[:]) + } + + if tokenExpiresAt.IsZero() { + // a cache slot is freed only by expiry, so a token without exp + // would hold it forever and eventually exhaust the bound + return fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims) + } + expiresAt := tokenExpiresAt.Add(clockSkewTolerance) + if !time.Now().Before(expiresAt) { + return fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired) + } + + entry := &jtiEntry{ + subject: subject, + expiresAt: expiresAt, + } + + // The read lock pins the store/count pair for the whole test-and-set + // so a replay-state handoff cannot repoint them mid-check and strand + // this consume in an abandoned store. + c.storeMu.RLock() + store, count := c.store, c.count + + // Atomic test-and-set detects replays before any capacity work runs, + // so a replayed token cannot trigger cleanup scans. + if _, loaded := store.LoadOrStore(key, entry); loaded { + c.storeMu.RUnlock() + glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)", + c.name, jtiLogKey(key)) + return fmt.Errorf("%w: jti %s has already been used", + providers.ErrProviderTokenReplayed, jtiLogKey(key)) + } + + // The count is incremented as part of the insert, so concurrent + // validations cannot all slip under the limit: whoever pushes the + // count over evicts their own entry and is rejected. + newCount := count.Add(1) + maxSize := c.maxSize.Load() + if newCount > maxSize { + // Over the bound: expired entries may still occupy slots, so + // reclaim them with a synchronized sweep before rejecting. + c.sweepStore(store, count) + if count.Load() > maxSize { + if _, loaded := store.LoadAndDelete(key); loaded { + count.Add(-1) + } + c.storeMu.RUnlock() + glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+ + "This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.", + c.name, maxSize) + return fmt.Errorf("JTI cache capacity exceeded") + } + } else if newCount >= maxSize*9/10 { + glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup", + c.name, (newCount*100)/maxSize, newCount, maxSize) + go c.performEagerCleanup() + } + c.storeMu.RUnlock() + + glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)", + c.name, jtiLogKey(key), expiresAt, newCount) + return nil +} + +// evictExpired removes entries whose expiry has passed and returns how +// many it deleted. Caller must hold sweepMu. +func (c *jtiReplayCache) evictExpired(now time.Time, store *sync.Map, count *atomic.Int64) int { + removed := 0 + store.Range(func(key, value interface{}) bool { + expiresAt := value.(*jtiEntry).expiresAt + if !expiresAt.IsZero() && now.After(expiresAt) { + // LoadAndDelete reports whether this call actually removed the + // entry, so concurrent deletes cannot decrement the count twice. + if v, loaded := store.LoadAndDelete(key); loaded { + if !v.(*jtiEntry).uncounted { + count.Add(-1) + } + removed++ + } + } + return true + }) + return removed +} + +// state returns the cache's current store/count pair. +func (c *jtiReplayCache) state() (*sync.Map, *atomic.Int64) { + c.storeMu.RLock() + defer c.storeMu.RUnlock() + return c.store, c.count +} + +// sweepStore runs one serialized expiry scan over the given store. It only +// takes sweepMu, so callers already holding storeMu may use it. +func (c *jtiReplayCache) sweepStore(store *sync.Map, count *atomic.Int64) { + c.sweepMu.Lock() + defer c.sweepMu.Unlock() + c.lastSweep.Store(time.Now().UnixNano()) + if removed := c.evictExpired(time.Now(), store, count); removed > 0 { + glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)", + c.name, removed, count.Load()) + } +} + +// sweepExpired runs a scan that is guaranteed to complete before the +// caller proceeds; the periodic worker uses it so the scan always covers +// the store currently in effect, not one captured earlier. +func (c *jtiReplayCache) sweepExpired() { + store, count := c.state() + c.sweepStore(store, count) +} + +// performEagerCleanup performs an opportunistic cleanup of expired JTI +// entries when the cache approaches capacity. At most one sweep runs at a +// time and no more often than eagerCleanupMinInterval, so a burst of +// requests near the limit cannot each turn into a full-cache scan. +func (c *jtiReplayCache) performEagerCleanup() { + if !c.enabled.Load() { + return + } + if time.Since(time.Unix(0, c.lastSweep.Load())) < eagerCleanupMinInterval { + return + } + store, count := c.state() + if !c.sweepMu.TryLock() { + return + } + defer c.sweepMu.Unlock() + c.lastSweep.Store(time.Now().UnixNano()) + + beforeCount := count.Load() + if removed := c.evictExpired(time.Now(), store, count); removed > 0 { + glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)", + c.name, removed, beforeCount, count.Load()) + } else { + glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+ + "Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.", + c.name) + } +} + +// cleanupLoop periodically removes expired JTI entries from the cache. +// This prevents memory growth by removing JTIs after their tokens have +// expired. Runs until ctx is cancelled; done is closed on exit. +func (c *jtiReplayCache) cleanupLoop(ctx context.Context, done chan struct{}, interval time.Duration) { + defer close(done) + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", c.name, interval) + + for { + select { + case <-ctx.Done(): + glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", c.name) + return + case <-ticker.C: + c.sweepExpired() + } + } +} + +// stop gracefully stops the cleanup worker. +// Should be called when the provider is being shut down to prevent goroutine +// leaks. +func (c *jtiReplayCache) stop(ctx context.Context) error { + c.mu.Lock() + cancel, done := c.cleanupCancel, c.cleanupDone + c.mu.Unlock() + + if cancel == nil { + return nil + } + cancel() + + select { + case <-done: + glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", c.name) + case <-ctx.Done(): + glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", c.name, ctx.Err()) + return ctx.Err() + } + + c.mu.Lock() + if c.cleanupDone == done { + c.cleanupCancel = nil + c.cleanupDone = nil + } + c.mu.Unlock() + return nil +} + +// adoptFrom takes over the consumed-token store of another cache, so tokens +// it already saw stay rejected across the rebuild. The caller must shut down +// the replaced provider; only one cleanup worker may sweep the shared store +// at a time. Call before the new provider serves validators; sweepMu +// excludes its own cleanup worker during the swap. +func (c *jtiReplayCache) adoptFrom(old *jtiReplayCache) { + if old == nil || old == c { + return + } + c.storeMu.Lock() + c.sweepMu.Lock() + c.store, c.count = old.store, old.count + c.sweepMu.Unlock() + c.storeMu.Unlock() +} + +// mergeFrom hands this cache's store to another cache and then folds the +// other's consumed-token entries in, so rejections the other recorded keep +// holding when its records move to a shared provider. Repointing first means +// tokens the other still consumes — in-flight exchanges or ones routed to it +// before the provider map updates — land in the shared store instead of a +// store this merge already drained. Unlike adoptFrom it swaps nothing of its +// own, so it is safe on a cache that is already serving validators. +func (c *jtiReplayCache) mergeFrom(other *jtiReplayCache) { + if other == nil || other == c { + return + } + c.storeMu.RLock() + defer c.storeMu.RUnlock() + store, count := c.store, c.count + + other.storeMu.Lock() + oldStore := other.store + if oldStore == store { + other.storeMu.Unlock() + return + } + other.store, other.count = store, count + other.storeMu.Unlock() + + c.sweepMu.Lock() + defer c.sweepMu.Unlock() + now := time.Now() + oldStore.Range(func(key, value interface{}) bool { + entry := value.(*jtiEntry) + if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) { + return true + } + // A consumed token is never dropped to fit the bound: merged entries + // stay but do not charge the count, so a store filled past the bound + // by the merge cannot reject new exchanges while it drains by expiry. + store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true}) + return true + }) +} + +// jtiLogKey redacts a token identifier for logs and error strings: a stable +// fingerprint that still lets operators correlate repeated presentations. +func jtiLogKey(jti string) string { + sum := sha256.Sum256([]byte(jti)) + return hex.EncodeToString(sum[:8]) +} diff --git a/weed/iam/oidc/oidc_provider.go b/weed/iam/oidc/oidc_provider.go index c2ccf4ab8f..6b115041d5 100644 --- a/weed/iam/oidc/oidc_provider.go +++ b/weed/iam/oidc/oidc_provider.go @@ -6,7 +6,6 @@ import ( "crypto/elliptic" "crypto/rsa" "crypto/sha1" - "crypto/sha256" "crypto/tls" "crypto/x509" "encoding/base64" @@ -20,7 +19,6 @@ import ( "path/filepath" "strings" "sync" - "sync/atomic" "time" "github.com/golang-jwt/jwt/v5" @@ -29,30 +27,6 @@ import ( "github.com/seaweedfs/seaweedfs/weed/security" ) -const ( - // clockSkewTolerance matches the typical JWT library clock skew allowance (RFC 7519) - clockSkewTolerance = 5 * time.Minute - // defaultJTICacheSize is the default maximum number of JTI entries to cache - defaultJTICacheSize = 10000 - // defaultCleanupInterval is how often to scan for expired JTI entries - defaultCleanupInterval = 5 * time.Minute - // eagerCleanupMinInterval rate limits opportunistic sweeps near capacity - eagerCleanupMinInterval = 5 * time.Second - // tokenFingerprintPrefix marks replay-cache keys derived from the token's - // own fingerprint, used when the token carries no usable jti claim - tokenFingerprintPrefix = "fp:" -) - -// jtiEntry represents a cached JTI with metadata -type jtiEntry struct { - subject string - expiresAt time.Time - // uncounted marks an entry merged in past the capacity bound: it keeps - // rejecting replays but does not charge jtiCount, so merged excess can - // neither reject new exchanges nor be double-counted when swept. - uncounted bool -} - // OIDCProvider implements OpenID Connect authentication with JTI replay protection. // // Replay Protection: @@ -84,31 +58,24 @@ type OIDCProvider struct { resolvedJWKSUri string discoveryFailed bool - // JTI replay protection fields. jtiStore and jtiCount are allocated once - // in NewOIDCProvider and survive re-initialization, so a config refresh - // cannot reopen already-consumed tokens for replay. - jtiEnabled atomic.Bool - jtiStore *sync.Map // map[string]*jtiEntry - atomic storage for consumed token keys - jtiCount *atomic.Int64 // entries in jtiStore; shared with the store - jtiMaxSize atomic.Int64 - jtiLastSweep atomic.Int64 // unix nano of the last sweep; rate limits eager cleanups + // jti wraps the consumed-token cache behind the replay protection. It is + // allocated by NewOIDCProvider or lazily by replayCache, and survives + // re-initialization, so a config refresh cannot reopen already-consumed + // tokens for replay. jtiOnce makes a zero-value OIDCProvider safe. + jti *jtiReplayCache + jtiOnce sync.Once +} - // jtiStoreMu guards the jtiStore/jtiCount pair: replay-state handoffs - // (AdoptReplayStateFrom, MergeReplayStateFrom) repoint them under the - // write lock while the consume path holds a read lock, so a consume can - // never land in a store the handoff already drained and abandoned. - // It is always acquired before jtiSweepMu, never while holding it. - jtiStoreMu sync.RWMutex - - // jtiMu guards the lifecycle of the cleanup worker: jtiCleanupCancel and - // jtiCleanupDone. The worker itself only touches jtiStore/jtiCount, so - // holding jtiMu while waiting for it is safe. - jtiMu sync.Mutex - jtiCleanupCancel context.CancelFunc - jtiCleanupDone chan struct{} // Closed when cleanup goroutine exits - - // jtiSweepMu serializes cache sweeps so at most one scan runs at a time. - jtiSweepMu sync.Mutex +// replayCache returns the consumed-token cache, allocating it for providers +// built without NewOIDCProvider so the delegated operations never panic on a +// nil cache. +func (p *OIDCProvider) replayCache() *jtiReplayCache { + p.jtiOnce.Do(func() { + if p.jti == nil { + p.jti = newJTIReplayCache(p.name) + } + }) + return p.jti } // OIDCConfig holds OIDC provider configuration @@ -391,8 +358,7 @@ func NewOIDCProvider(name string) *OIDCProvider { return &OIDCProvider{ name: name, httpClient: &http.Client{Timeout: 30 * time.Second, CheckRedirect: refuseDowngradeRedirect}, - jtiStore: &sync.Map{}, - jtiCount: &atomic.Int64{}, + jti: newJTIReplayCache(name), } } @@ -512,53 +478,12 @@ func (p *OIDCProvider) Initialize(config interface{}) error { if oidcConfig.JTIReplayProtectionEnabled != nil { jtiEnabled = *oidcConfig.JTIReplayProtectionEnabled } - - p.jtiMu.Lock() - p.stopJTICleanupWorkerLocked() - if jtiEnabled { - maxSize := int64(oidcConfig.JTICacheMaxSize) - if maxSize <= 0 { - maxSize = defaultJTICacheSize - } - // the bound must land before the flag: a consume that sees enabled - // with a zero bound would reject the first token it registers - p.jtiMaxSize.Store(maxSize) - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - p.jtiCleanupCancel = cancel - p.jtiCleanupDone = done - p.jtiEnabled.Store(true) - p.jtiMu.Unlock() - - go p.cleanupExpiredJTIs(ctx, done, defaultCleanupInterval) - glog.V(2).Infof("OIDC provider %q: JTI replay protection enabled (max size: %d)", p.name, maxSize) - } else { - p.jtiEnabled.Store(false) - p.jtiMu.Unlock() - glog.Warningf("OIDC provider %q: JTI replay protection is DISABLED", p.name) - } + p.replayCache().configure(jtiEnabled, int64(oidcConfig.JTICacheMaxSize)) // For testing, we'll skip the actual OIDC client initialization return nil } -// stopJTICleanupWorkerLocked cancels the current cleanup goroutine and waits -// briefly for it to exit. Caller must hold jtiMu. -func (p *OIDCProvider) stopJTICleanupWorkerLocked() { - if p.jtiCleanupCancel == nil { - return - } - p.jtiCleanupCancel() - select { - case <-p.jtiCleanupDone: - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine exited", p.name) - case <-time.After(2 * time.Second): - glog.Warningf("OIDC provider %q: Timeout waiting for JTI cleanup goroutine", p.name) - } - p.jtiCleanupCancel = nil - p.jtiCleanupDone = nil -} - // validateConfig validates the OIDC configuration func (p *OIDCProvider) validateConfig(config *OIDCConfig) error { if config.Issuer == "" { @@ -935,85 +860,14 @@ func (p *OIDCProvider) validateToken(ctx context.Context, token string, singleUs tokenClaims.Claims[key] = value } - // Extract and validate JTI claim for replay protection. The check runs - // only for single-use callers: a bearer token is meant to be presented on - // every request, so validating it must not consume it. - if singleUse && p.jtiEnabled.Load() { + // Consume the token for replay protection. The check runs only for + // single-use callers: a bearer token is meant to be presented on every + // request, so validating it must not consume it. + if singleUse { jti, _ := claims["jti"].(string) - key := jti - if key == "" { - // Providers that issue no jti cannot be tracked by claim, so the - // signed input keys the entry: a repeated presentation of the same - // claims is a replay even when the signature bytes differ (e.g. - // ECDSA s/N-s malleability). - signingInput := token - if i := strings.LastIndexByte(token, '.'); i > 0 { - signingInput = token[:i] - } - sum := sha256.Sum256([]byte(signingInput)) - key = tokenFingerprintPrefix + hex.EncodeToString(sum[:]) + if err := p.replayCache().consume(token, jti, subject, tokenClaims.ExpiresAt); err != nil { + return nil, err } - - expiresAt := tokenClaims.ExpiresAt - if expiresAt.IsZero() { - // a cache slot is freed only by expiry, so a token without exp - // would hold it forever and eventually exhaust the bound - return nil, fmt.Errorf("%w: missing exp claim", providers.ErrProviderMissingClaims) - } - expiresAt = expiresAt.Add(clockSkewTolerance) - if !time.Now().Before(expiresAt) { - return nil, fmt.Errorf("%w: token expired", providers.ErrProviderTokenExpired) - } - - entry := &jtiEntry{ - subject: subject, - expiresAt: expiresAt, - } - - // The read lock pins the store/count pair for the whole test-and-set - // so a replay-state handoff cannot repoint them mid-check and strand - // this consume in an abandoned store. - p.jtiStoreMu.RLock() - store, count := p.jtiStore, p.jtiCount - - // Atomic test-and-set detects replays before any capacity work runs, - // so a replayed token cannot trigger cleanup scans. - if _, loaded := store.LoadOrStore(key, entry); loaded { - p.jtiStoreMu.RUnlock() - glog.Warningf("OIDC provider %q: Token replay detected (jti: %s)", - p.name, jtiLogKey(key)) - return nil, fmt.Errorf("%w: jti %s has already been used", - providers.ErrProviderTokenReplayed, jtiLogKey(key)) - } - - // The count is incremented as part of the insert, so concurrent - // validations cannot all slip under the limit: whoever pushes the - // count over evicts their own entry and is rejected. - newCount := count.Add(1) - maxSize := p.jtiMaxSize.Load() - if newCount > maxSize { - // Over the bound: expired entries may still occupy slots, so - // reclaim them with a synchronized sweep before rejecting. - p.sweepJTIStore(store, count) - if count.Load() > maxSize { - if _, loaded := store.LoadAndDelete(key); loaded { - count.Add(-1) - } - p.jtiStoreMu.RUnlock() - glog.Errorf("OIDC provider %q: JTI cache FULL (%d entries) - rejecting token. "+ - "This may indicate a DoS attack or insufficient cache size. Consider increasing JTICacheMaxSize.", - p.name, maxSize) - return nil, fmt.Errorf("JTI cache capacity exceeded") - } - } else if newCount >= maxSize*9/10 { - glog.Warningf("OIDC provider %q: JTI cache at %d%% capacity (%d/%d), triggering eager cleanup", - p.name, (newCount*100)/maxSize, newCount, maxSize) - go p.performEagerCleanup() - } - p.jtiStoreMu.RUnlock() - - glog.V(4).Infof("OIDC provider %q: Registered JTI %s (expires at %v, cache size: %d)", - p.name, jtiLogKey(key), expiresAt, newCount) } return tokenClaims, nil @@ -1445,148 +1299,22 @@ func (p *OIDCProvider) mapUserInfoToIdentity(userInfo map[string]interface{}) *p return identity } -// evictExpiredJTIs removes entries whose expiry has passed and returns how -// many it deleted. Caller must hold jtiSweepMu. -func (p *OIDCProvider) evictExpiredJTIs(now time.Time, store *sync.Map, count *atomic.Int64) int { - removed := 0 - store.Range(func(key, value interface{}) bool { - expiresAt := value.(*jtiEntry).expiresAt - if !expiresAt.IsZero() && now.After(expiresAt) { - // LoadAndDelete reports whether this call actually removed the - // entry, so concurrent deletes cannot decrement the count twice. - if v, loaded := store.LoadAndDelete(key); loaded { - if !v.(*jtiEntry).uncounted { - count.Add(-1) - } - removed++ - } - } - return true - }) - return removed -} - -// replayState returns the provider's current store/count pair. -func (p *OIDCProvider) replayState() (*sync.Map, *atomic.Int64) { - p.jtiStoreMu.RLock() - defer p.jtiStoreMu.RUnlock() - return p.jtiStore, p.jtiCount -} - -// sweepJTIStore runs one serialized expiry scan over the given store. It only -// takes jtiSweepMu, so callers already holding jtiStoreMu may use it. -func (p *OIDCProvider) sweepJTIStore(store *sync.Map, count *atomic.Int64) { - p.jtiSweepMu.Lock() - defer p.jtiSweepMu.Unlock() - p.jtiLastSweep.Store(time.Now().UnixNano()) - if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { - glog.V(4).Infof("OIDC provider %q: Cleaned up %d expired JTI entries (remaining: %d)", - p.name, removed, count.Load()) - } -} - -// sweepExpiredJTIs runs a scan that is guaranteed to complete before the -// caller proceeds; the capacity-rejection path uses it so a fresh token is -// not refused while expired entries still hold slots. -func (p *OIDCProvider) sweepExpiredJTIs() { - store, count := p.replayState() - p.sweepJTIStore(store, count) -} - -// performEagerCleanup performs an opportunistic cleanup of expired JTI -// entries when the cache approaches capacity. At most one sweep runs at a -// time and no more often than eagerCleanupMinInterval, so a burst of -// requests near the limit cannot each turn into a full-cache scan. -func (p *OIDCProvider) performEagerCleanup() { - if !p.jtiEnabled.Load() { - return - } - if time.Since(time.Unix(0, p.jtiLastSweep.Load())) < eagerCleanupMinInterval { - return - } - store, count := p.replayState() - if !p.jtiSweepMu.TryLock() { - return - } - defer p.jtiSweepMu.Unlock() - p.jtiLastSweep.Store(time.Now().UnixNano()) - - beforeCount := count.Load() - if removed := p.evictExpiredJTIs(time.Now(), store, count); removed > 0 { - glog.Infof("OIDC provider %q: Eager cleanup removed %d expired JTI entries (%d -> %d)", - p.name, removed, beforeCount, count.Load()) - } else { - glog.Warningf("OIDC provider %q: Eager cleanup found no expired entries to remove. "+ - "Cache may be filled with valid long-lived tokens. Consider increasing JTICacheMaxSize.", - p.name) - } -} - -// cleanupExpiredJTIs periodically removes expired JTI entries from the cache. -// This prevents memory growth by removing JTIs after their tokens have expired. -// Runs in a background goroutine until ctx is cancelled; done is closed on exit. -func (p *OIDCProvider) cleanupExpiredJTIs(ctx context.Context, done chan struct{}, interval time.Duration) { - defer close(done) - - ticker := time.NewTicker(interval) - defer ticker.Stop() - - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine started (interval: %v)", p.name, interval) - - for { - select { - case <-ctx.Done(): - glog.V(3).Infof("OIDC provider %q: JTI cleanup goroutine stopping", p.name) - return - case <-ticker.C: - p.sweepExpiredJTIs() - } - } -} - // Shutdown gracefully stops the JTI cleanup goroutine. // Should be called when the provider is being shut down to prevent goroutine leaks. func (p *OIDCProvider) Shutdown(ctx context.Context) error { - p.jtiMu.Lock() - cancel, done := p.jtiCleanupCancel, p.jtiCleanupDone - p.jtiMu.Unlock() - - if cancel == nil { - return nil - } - cancel() - - select { - case <-done: - glog.V(2).Infof("OIDC provider %q: Cleanup goroutine shutdown complete", p.name) - case <-ctx.Done(): - glog.Warningf("OIDC provider %q: Shutdown timeout waiting for cleanup goroutine: %v", p.name, ctx.Err()) - return ctx.Err() - } - - p.jtiMu.Lock() - if p.jtiCleanupDone == done { - p.jtiCleanupCancel = nil - p.jtiCleanupDone = nil - } - p.jtiMu.Unlock() - return nil + return p.replayCache().stop(ctx) } // AdoptReplayStateFrom takes over the consumed-token store of the provider it // replaces, so tokens it already saw stay rejected across the rebuild. The // caller must shut down the replaced provider; only one cleanup worker may // sweep the shared store at a time. Call before the new provider serves -// validators; jtiSweepMu excludes its own cleanup worker during the swap. +// validators. func (p *OIDCProvider) AdoptReplayStateFrom(old *OIDCProvider) { - if old == nil || old == p { + if old == nil || old == p || old.jti == nil { return } - p.jtiStoreMu.Lock() - p.jtiSweepMu.Lock() - p.jtiStore, p.jtiCount = old.jtiStore, old.jtiCount - p.jtiSweepMu.Unlock() - p.jtiStoreMu.Unlock() + p.replayCache().adoptFrom(old.jti) } // MergeReplayStateFrom hands this provider's store to another provider and @@ -1601,38 +1329,5 @@ func (p *OIDCProvider) MergeReplayStateFrom(other *OIDCProvider) { if other == nil || other == p { return } - p.jtiStoreMu.RLock() - defer p.jtiStoreMu.RUnlock() - store, count := p.jtiStore, p.jtiCount - - other.jtiStoreMu.Lock() - oldStore := other.jtiStore - if oldStore == store { - other.jtiStoreMu.Unlock() - return - } - other.jtiStore, other.jtiCount = store, count - other.jtiStoreMu.Unlock() - - p.jtiSweepMu.Lock() - defer p.jtiSweepMu.Unlock() - now := time.Now() - oldStore.Range(func(key, value interface{}) bool { - entry := value.(*jtiEntry) - if expiresAt := entry.expiresAt; !expiresAt.IsZero() && now.After(expiresAt) { - return true - } - // A consumed token is never dropped to fit the bound: merged entries - // stay but do not charge the count, so a store filled past the bound - // by the merge cannot reject new exchanges while it drains by expiry. - store.LoadOrStore(key, &jtiEntry{subject: entry.subject, expiresAt: entry.expiresAt, uncounted: true}) - return true - }) -} - -// jtiLogKey redacts a token identifier for logs and error strings: a stable -// fingerprint that still lets operators correlate repeated presentations. -func jtiLogKey(jti string) string { - sum := sha256.Sum256([]byte(jti)) - return hex.EncodeToString(sum[:8]) + p.replayCache().mergeFrom(other.replayCache()) } diff --git a/weed/iam/oidc/oidc_provider_test.go b/weed/iam/oidc/oidc_provider_test.go index 3744ec3138..29f7e3e656 100644 --- a/weed/iam/oidc/oidc_provider_test.go +++ b/weed/iam/oidc/oidc_provider_test.go @@ -1067,10 +1067,10 @@ func TestOIDCProviderJTICleanup(t *testing.T) { require.NoError(t, err) // Check cache size is 1 - assert.Equal(t, int64(1), provider.jtiCount.Load()) + assert.Equal(t, int64(1), provider.jti.count.Load()) // Verify entry exists and check its expiration includes clock skew - val, ok := provider.jtiStore.Load("cleanup-test-jti") + val, ok := provider.jti.store.Load("cleanup-test-jti") require.True(t, ok, "JTI entry should exist") entry := val.(*jtiEntry) @@ -1088,21 +1088,21 @@ func TestOIDCProviderJTICleanup(t *testing.T) { // both the entry and the count must actually be cleared cleanupCtx, cleanupCancel := context.WithCancel(context.Background()) cleanupDone := make(chan struct{}) - go provider.cleanupExpiredJTIs(cleanupCtx, cleanupDone, 20*time.Millisecond) + go provider.jti.cleanupLoop(cleanupCtx, cleanupDone, 20*time.Millisecond) defer func() { cleanupCancel() <-cleanupDone }() - provider.jtiStore.Store("expired-cleanup-jti", &jtiEntry{ + provider.jti.store.Store("expired-cleanup-jti", &jtiEntry{ subject: "expired-user", expiresAt: time.Now().Add(-time.Minute), }) - provider.jtiCount.Add(1) + provider.jti.count.Add(1) require.Eventually(t, func() bool { - _, stillThere := provider.jtiStore.Load("expired-cleanup-jti") - return !stillThere && provider.jtiCount.Load() == 1 // only the valid entry remains + _, stillThere := provider.jti.store.Load("expired-cleanup-jti") + return !stillThere && provider.jti.count.Load() == 1 // only the valid entry remains }, 2*time.Second, 10*time.Millisecond, "periodic cleanup should evict the expired entry") } @@ -1124,7 +1124,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // First initialization require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "First cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "First cleanup cancel should be set") // Create and validate a token to ensure JTI protection is working token1 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1140,7 +1140,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // Second initialization should not cause errors require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "Second cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "Second cleanup cancel should be set") // The JTI cache survives re-initialization, so the consumed token is // still detected as a replay rather than being usable again @@ -1150,7 +1150,7 @@ func TestOIDCProviderReinitialization(t *testing.T) { // Third initialization to ensure it works multiple times require.NoError(t, provider.Initialize(config)) - require.NotNil(t, provider.jtiCleanupCancel, "Third cleanup cancel should be set") + require.NotNil(t, provider.jti.cleanupCancel, "Third cleanup cancel should be set") // Verify JTI replay protection still works after re-initialization token2 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1201,16 +1201,16 @@ func TestOIDCProviderJTIEagerCleanup(t *testing.T) { subject: fmt.Sprintf("expired-user-%d", i), expiresAt: time.Now().Add(-time.Hour), // Already expired } - provider.jtiStore.Store(jti, entry) - provider.jtiCount.Add(1) + provider.jti.store.Store(jti, entry) + provider.jti.count.Add(1) } - assert.Equal(t, int64(5), provider.jtiCount.Load(), "Should have 5 expired entries") + assert.Equal(t, int64(5), provider.jti.count.Load(), "Should have 5 expired entries") // Perform eager cleanup - should remove all expired entries - provider.performEagerCleanup() + provider.jti.performEagerCleanup() - assert.Equal(t, int64(0), provider.jtiCount.Load(), + assert.Equal(t, int64(0), provider.jti.count.Load(), "All expired entries should be removed by eager cleanup") // Now test that adding valid tokens works after cleanup @@ -1228,7 +1228,7 @@ func TestOIDCProviderJTIEagerCleanup(t *testing.T) { } // Should have 9 valid entries now (90% of capacity, triggers eager cleanup warning) - assert.Equal(t, int64(9), provider.jtiCount.Load(), "Should have 9 valid entries") + assert.Equal(t, int64(9), provider.jti.count.Load(), "Should have 9 valid entries") // 10th token should succeed (at capacity but not over) token10 := createTestJWT(t, privateKey, jwt.MapClaims{ @@ -1282,11 +1282,11 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { subject: fmt.Sprintf("user-%d", i), expiresAt: time.Now().Add(-time.Hour), // Already expired } - provider.jtiStore.Store(jti, entry) - provider.jtiCount.Add(1) + provider.jti.store.Store(jti, entry) + provider.jti.count.Add(1) } - initialCount := provider.jtiCount.Load() + initialCount := provider.jti.count.Load() assert.Equal(t, int64(50), initialCount, "Should have 50 expired entries") // Run multiple concurrent cleanups @@ -1297,14 +1297,14 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - provider.performEagerCleanup() + provider.jti.performEagerCleanup() }() } wg.Wait() // Counter should be 0 (all entries removed), never negative - finalCount := provider.jtiCount.Load() + finalCount := provider.jti.count.Load() assert.Equal(t, int64(0), finalCount, "Counter should be 0 after concurrent cleanups, got %d", finalCount) assert.GreaterOrEqual(t, finalCount, int64(0), @@ -1312,7 +1312,7 @@ func TestOIDCProviderConcurrentCleanupNoNegativeCount(t *testing.T) { // Verify store is actually empty storeCount := 0 - provider.jtiStore.Range(func(key, value interface{}) bool { + provider.jti.store.Range(func(key, value interface{}) bool { storeCount++ return true }) From de0215821242c751bf373582c63bc453b392d918 Mon Sep 17 00:00:00 2001 From: jnicolasmg <94394309+niko4464@users.noreply.github.com> Date: Sat, 24 Oct 2026 10:03:47 -0400 Subject: [PATCH 56/70] filer: let NewChunkStreamReaderFromFiler fall back to the master on a cache miss (#11964) refreshUrls invalidates every cached location of a volume after a failed read and re-looks it up through the reader's lookup function. With the cache-only masterClient.LookupFileId, one failed read to one replica left the volume missing from the filer's cache until restart, so metadata-log replay skipped it. Use GetLookupFileIdFunction like the other read paths. Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Chris Lu --- weed/filer/stream.go | 6 +- weed/filer/stream_from_filer_lookup_test.go | 101 ++++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) create mode 100644 weed/filer/stream_from_filer_lookup_test.go diff --git a/weed/filer/stream.go b/weed/filer/stream.go index a34a6c548f..260c6f5c24 100644 --- a/weed/filer/stream.go +++ b/weed/filer/stream.go @@ -415,11 +415,7 @@ func doNewChunkStreamReader(ctx context.Context, lookupFileIdFn wdclient.LookupF func NewChunkStreamReaderFromFiler(ctx context.Context, masterClient *wdclient.MasterClient, chunks []*filer_pb.FileChunk) *ChunkStreamReader { - lookupFileIdFn := func(ctx context.Context, fileId string) (targetUrl []string, err error) { - return masterClient.LookupFileId(ctx, fileId) - } - - return doNewChunkStreamReader(ctx, lookupFileIdFn, chunks) + return doNewChunkStreamReader(ctx, masterClient.GetLookupFileIdFunction(), chunks) } // NewChunkStreamReaderFromLookup creates a ChunkStreamReader from a lookup function. diff --git a/weed/filer/stream_from_filer_lookup_test.go b/weed/filer/stream_from_filer_lookup_test.go new file mode 100644 index 0000000000..51d604a4a6 --- /dev/null +++ b/weed/filer/stream_from_filer_lookup_test.go @@ -0,0 +1,101 @@ +package filer + +import ( + "context" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + + "github.com/seaweedfs/seaweedfs/weed/cluster" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/wdclient" +) + +// lookupOnlyMaster announces no volumes on KeepConnected, so the client's +// cache stays empty, but answers LookupVolume with the one volume server. +type lookupOnlyMaster struct { + master_pb.UnimplementedSeaweedServer + volumeUrl string +} + +func (m *lookupOnlyMaster) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error { + if _, err := stream.Recv(); err != nil { + return err + } + if err := stream.Send(&master_pb.KeepConnectedResponse{}); err != nil { + return err + } + <-stream.Context().Done() + return nil +} + +func (m *lookupOnlyMaster) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) { + resp := &master_pb.LookupVolumeResponse{} + for _, id := range req.VolumeOrFileIds { + resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{ + VolumeOrFileId: id, + Locations: []*master_pb.Location{{Url: m.volumeUrl, PublicUrl: m.volumeUrl}}, + }) + } + return resp, nil +} + +// A filer's cached volume map can lose a volume that still exists: a failed +// read invalidates every cached location of it, and the master only announces +// changes. Readers built from the filer's MasterClient must then ask the master +// instead of failing until the process restarts. +func TestChunkStreamReaderFromFilerAsksMasterOnCacheMiss(t *testing.T) { + content := []byte("metadata log chunk") + volume := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(content) + })) + t.Cleanup(volume.Close) + + master := &lookupOnlyMaster{volumeUrl: strings.TrimPrefix(volume.URL, "http://")} + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + grpcServer := grpc.NewServer() + master_pb.RegisterSeaweedServer(grpcServer, master) + go func() { _ = grpcServer.Serve(lis) }() + t.Cleanup(grpcServer.Stop) + _, port, err := net.SplitHostPort(lis.Addr().String()) + if err != nil { + t.Fatalf("split listener address: %v", err) + } + + mc := wdclient.NewMasterClient( + grpc.WithTransportCredentials(insecure.NewCredentials()), + "", cluster.FilerType, pb.ServerAddress("localhost:0"), "", "", + *pb.NewServiceDiscoveryFromMap(map[string]pb.ServerAddress{"m": pb.ServerAddress(fmt.Sprintf("127.0.0.1:0.%s", port))}), + ) + connecting, stopConnecting := context.WithCancel(context.Background()) + t.Cleanup(stopConnecting) + go mc.KeepConnectedToMaster(connecting) + waiting, cancelWaiting := context.WithTimeout(context.Background(), 20*time.Second) + defer cancelWaiting() + mc.WaitUntilConnected(waiting) + if waiting.Err() != nil { + t.Fatal("the master client never connected to the fake master") + } + + chunks := []*filer_pb.FileChunk{{FileId: "1,0123456789", Offset: 0, Size: uint64(len(content))}} + got, err := io.ReadAll(NewChunkStreamReaderFromFiler(context.Background(), mc, chunks)) + if err != nil { + t.Fatalf("read with volume 1 missing from the cache: %v", err) + } + if string(got) != string(content) { + t.Fatalf("read %q, want %q", got, content) + } +} From b0e6f7a096609aa21d9a4eaa954e3f042d1ade78 Mon Sep 17 00:00:00 2001 From: Wleeaf Date: Sat, 24 Oct 2026 17:04:01 +0300 Subject: [PATCH 57/70] s3: allow up to 50 tags in a bucket tag set (#12030) * s3: allow up to 50 tags in a bucket tag set PutBucketTagging validated the tag set with the object tag rules, which allow 10 tags. AWS allows a bucket up to 50 tags, so tag sets of 11 to 50 tags were rejected with InvalidTag. Validate bucket tags against 50 and keep the limit of 10 for objects and S3 Tables. A tag set that repeats a key is rejected with InvalidTag, as AWS does, instead of keeping one of the values, so 51 tags with a repeated key no longer pass as 50. * s3: reject duplicate tag keys in PutObjectTagging The tag set collapses to a map, so a repeated key was silently kept once.\nMatch the bucket tagging check and answer InvalidTag instead. * s3: reject empty tag keys AWS requires a tag key to be at least one character; the shared\nvalidator only checked the 128-char upper bound. * s3: reject duplicate tag keys in the tagging header parser parseTagsHeader (the x-amz-tagging path used by CopyObject with TaggingDirective=REPLACE) silently kept the last value when a key repeated, while the PutObject and multipart header paths and the new XML checks return InvalidTag for the same input. A repeated key after URL decoding now fails the same way. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: validate x-amz-tagging tags on PutObject before storing the entry The header path only rejected duplicate keys, so an upload could store more than ten tags or keys and values that fail S3 tag rules. Run the parsed set through ValidateTags like the XML tagging handlers do. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: validate x-amz-tagging tags on multipart initiate ParseS3Metadata stored header tags with no tag-set validation, the same gap putToFiler had. Multipart uploads now reject oversize or malformed tag sets with InvalidTag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: count tag length in characters, not bytes * s3: validate object tags before uploading chunks * s3: reuse parseTagsHeader and drop raw header from warnings --------- Co-authored-by: Wleeaf <179125857+wleeaf@users.noreply.github.com> Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/s3_metadata_util.go | 24 ++--- weed/s3api/s3api_bucket_tagging_handlers.go | 9 +- .../s3api_bucket_tagging_handlers_test.go | 91 +++++++++++++++++++ weed/s3api/s3api_object_handlers_put.go | 42 +++++---- weed/s3api/s3api_object_handlers_tagging.go | 6 ++ .../s3api_object_handlers_tagging_test.go | 29 ++++++ weed/s3api/s3tables/utils.go | 17 +++- weed/s3api/tags.go | 11 +++ weed/s3api/tags_test.go | 29 ++++++ 9 files changed, 215 insertions(+), 43 deletions(-) create mode 100644 weed/s3api/s3api_bucket_tagging_handlers_test.go create mode 100644 weed/s3api/s3api_object_handlers_tagging_test.go diff --git a/weed/s3api/s3_metadata_util.go b/weed/s3api/s3_metadata_util.go index bcd62b0e9d..01e0adc194 100644 --- a/weed/s3api/s3_metadata_util.go +++ b/weed/s3api/s3_metadata_util.go @@ -2,7 +2,6 @@ package s3api import ( "net/http" - "net/url" "strings" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -52,25 +51,16 @@ func ParseS3Metadata(r *http.Request, existing map[string][]byte, isReplace bool // Object tagging if tags := r.Header.Get(s3_constants.AmzObjectTagging); tags != "" { - // Use url.ParseQuery for robust parsing and automatic URL decoding - parsedTags, err := url.ParseQuery(tags) + tagSet, err := parseTagsHeader(tags) if err != nil { - // Return proper S3 error instead of silently dropping tags - glog.Warningf("Invalid S3 tag format in header '%s': %v", tags, err) + glog.Warningf("Invalid S3 tagging header: %v", err) return nil, s3err.ErrInvalidTag } - - // Validate: S3 spec does not allow duplicate tag keys - for key, values := range parsedTags { - if len(values) > 1 { - glog.Warningf("Duplicate tag key '%s' in header '%s'", key, tags) - return nil, s3err.ErrInvalidTag - } - // Tag value can be an empty string but not nil - value := "" - if len(values) > 0 { - value = values[0] - } + if err := ValidateTags(tagSet); err != nil { + glog.Warningf("Invalid tags (%d): %v", len(tagSet), err) + return nil, s3err.ErrInvalidTag + } + for key, value := range tagSet { metadata[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) } } diff --git a/weed/s3api/s3api_bucket_tagging_handlers.go b/weed/s3api/s3api_bucket_tagging_handlers.go index a1b116fd28..9541f38cb1 100644 --- a/weed/s3api/s3api_bucket_tagging_handlers.go +++ b/weed/s3api/s3api_bucket_tagging_handlers.go @@ -68,9 +68,14 @@ func (s3a *S3ApiServer) PutBucketTaggingHandler(w http.ResponseWriter, r *http.R } tags := tagging.ToTags() + // A key may appear only once in a tag set; the map keeps one of them. + if len(tags) != len(tagging.TagSet.Tag) { + glog.Errorf("PutBucketTagging duplicate tag keys %s", r.URL) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) + return + } - // Validate tags using existing validation - err = ValidateTags(tags) + err = ValidateBucketTags(tags) if err != nil { glog.Errorf("PutBucketTagging ValidateTags error %s: %v", r.URL, err) s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) diff --git a/weed/s3api/s3api_bucket_tagging_handlers_test.go b/weed/s3api/s3api_bucket_tagging_handlers_test.go new file mode 100644 index 0000000000..a1cc26b165 --- /dev/null +++ b/weed/s3api/s3api_bucket_tagging_handlers_test.go @@ -0,0 +1,91 @@ +package s3api + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" + "google.golang.org/protobuf/proto" +) + +// A bucket's tag set holds up to 50 tags, unlike an object's 10, and no key +// twice. +func TestPutBucketTaggingTagCountLimit(t *testing.T) { + const bucket = "tag-limit-bucket" + for _, tc := range []struct { + tags int + repeatKey bool // the last tag repeats the first tag's key + emptyKey bool // the first tag has an empty key + wantCode int + }{ + {tags: 10, wantCode: http.StatusOK}, + {tags: 11, wantCode: http.StatusOK}, + {tags: 50, wantCode: http.StatusOK}, + {tags: 51, wantCode: http.StatusBadRequest}, + {tags: 2, repeatKey: true, wantCode: http.StatusBadRequest}, + {tags: 51, repeatKey: true, wantCode: http.StatusBadRequest}, + {tags: 1, emptyKey: true, wantCode: http.StatusBadRequest}, + } { + name := fmt.Sprintf("%d tags", tc.tags) + if tc.repeatKey { + name += " with a repeated key" + } + if tc.emptyKey { + name += " with an empty key" + } + t.Run(name, func(t *testing.T) { + filer := &fakeTxnFiler{} + s3a := newHeadBucketTestServer(t, filer) + s3a.iam = &IdentityAccessManagement{isAuthEnabled: true} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set(bucket, &BucketConfig{Name: bucket}) + + var body strings.Builder + body.WriteString("") + for i := 0; i < tc.tags; i++ { + key := fmt.Sprintf("key%d", i) + if tc.repeatKey && i == tc.tags-1 { + key = "key0" + } + if tc.emptyKey && i == 0 { + key = "" + } + fmt.Fprintf(&body, "%svalue%d", key, i) + } + body.WriteString("") + + req := httptest.NewRequest(http.MethodPut, "/"+bucket+"?tagging", strings.NewReader(body.String())) + req = mux.SetURLVars(req, map[string]string{"bucket": bucket}) + rr := httptest.NewRecorder() + s3a.PutBucketTaggingHandler(rr, req) + + if rr.Code != tc.wantCode { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, tc.wantCode, rr.Body.String()) + } + if tc.wantCode != http.StatusOK { + if !strings.Contains(rr.Body.String(), "InvalidTag") { + t.Fatalf("body = %s, want InvalidTag", rr.Body.String()) + } + if filer.lastReq != nil { + t.Fatalf("rejected tag set was written to the filer") + } + return + } + if filer.lastReq == nil || len(filer.lastReq.Mutations) != 1 { + t.Fatalf("expected one bucket metadata write, got %+v", filer.lastReq) + } + var stored s3_pb.BucketMetadata + if err := proto.Unmarshal(filer.lastReq.Mutations[0].Content, &stored); err != nil { + t.Fatalf("decode stored bucket metadata: %v", err) + } + if len(stored.Tags) != tc.tags { + t.Fatalf("stored %d tags, want %d", len(stored.Tags), tc.tags) + } + }) + } +} diff --git a/weed/s3api/s3api_object_handlers_put.go b/weed/s3api/s3api_object_handlers_put.go index b38a205f16..aa6682712a 100644 --- a/weed/s3api/s3api_object_handlers_put.go +++ b/weed/s3api/s3api_object_handlers_put.go @@ -527,6 +527,22 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader if !s3_constants.IsValidBucketName(bucket) || (object != "" && !s3_constants.IsValidObjectKey(object)) { return "", s3err.ErrInvalidRequest, SSEResponseMetadata{} } + + // Reject invalid tags before any chunks are uploaded so a failed request + // leaves nothing behind on the volume servers. + var tagSet map[string]string + if tagging := r.Header.Get(s3_constants.AmzObjectTagging); tagging != "" { + parsed, err := parseTagsHeader(tagging) + if err != nil { + glog.Warningf("putToFiler: invalid tagging header for %s: %v", filePath, err) + return "", s3err.ErrInvalidTag, SSEResponseMetadata{} + } + if err := ValidateTags(parsed); err != nil { + glog.Warningf("putToFiler: invalid tags (%d) for %s: %v", len(parsed), filePath, err) + return "", s3err.ErrInvalidTag, SSEResponseMetadata{} + } + tagSet = parsed + } // NEW OPTIMIZATION: Write directly to volume servers, bypassing filer proxy // This eliminates the filer proxy overhead for PUT operations // Note: filePath is now passed directly instead of URL (no parsing needed) @@ -938,26 +954,12 @@ func (s3a *S3ApiServer) putToFiler(r *http.Request, filePath string, dataReader entry.Extended[s3_constants.AmzStorageClass] = []byte(sc) } - // Parse and store object tags from X-Amz-Tagging header - // Fix for GitHub issue #7589: Tags sent during object upload were not being stored - if tagging := r.Header.Get(s3_constants.AmzObjectTagging); tagging != "" { - parsedTags, err := url.ParseQuery(tagging) - if err != nil { - glog.Warningf("putToFiler: Invalid S3 tag format in header '%s': %v", tagging, err) - return "", s3err.ErrInvalidTag, SSEResponseMetadata{} - } - for key, values := range parsedTags { - if len(values) > 1 { - glog.Warningf("putToFiler: Duplicate tag key '%s' in header", key) - return "", s3err.ErrInvalidTag, SSEResponseMetadata{} - } - value := "" - if len(values) > 0 { - value = values[0] - } - entry.Extended[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) - } - glog.V(3).Infof("putToFiler: stored %d tags from X-Amz-Tagging header", len(parsedTags)) + // Store the tags validated before the chunk upload + for key, value := range tagSet { + entry.Extended[s3_constants.AmzObjectTagging+"-"+key] = []byte(value) + } + if len(tagSet) > 0 { + glog.V(3).Infof("putToFiler: stored %d tags from X-Amz-Tagging header", len(tagSet)) } storeSSEMetadata(entry, sseResult) diff --git a/weed/s3api/s3api_object_handlers_tagging.go b/weed/s3api/s3api_object_handlers_tagging.go index 466d8bcfd4..c55b38c09c 100644 --- a/weed/s3api/s3api_object_handlers_tagging.go +++ b/weed/s3api/s3api_object_handlers_tagging.go @@ -139,6 +139,12 @@ func (s3a *S3ApiServer) PutObjectTaggingHandler(w http.ResponseWriter, r *http.R return } tags := tagging.ToTags() + // A key may appear only once in a tag set; the map keeps one of them. + if len(tags) != len(tagging.TagSet.Tag) { + glog.Errorf("PutObjectTaggingHandler duplicate tag keys %s", r.URL) + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag) + return + } err = ValidateTags(tags) if err != nil { glog.Errorf("PutObjectTaggingHandler ValidateTags error %s: %v", r.URL, err) diff --git a/weed/s3api/s3api_object_handlers_tagging_test.go b/weed/s3api/s3api_object_handlers_tagging_test.go new file mode 100644 index 0000000000..05dec5a833 --- /dev/null +++ b/weed/s3api/s3api_object_handlers_tagging_test.go @@ -0,0 +1,29 @@ +package s3api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gorilla/mux" +) + +func TestPutObjectTaggingRejectsDuplicateKeys(t *testing.T) { + s3a := newHeadBucketTestServer(t, &fakeTxnFiler{}) + body := "" + + "a1" + + "a2" + + "" + req := httptest.NewRequest(http.MethodPut, "/b/o?tagging", strings.NewReader(body)) + req = mux.SetURLVars(req, map[string]string{"bucket": "b", "object": "o"}) + rr := httptest.NewRecorder() + s3a.PutObjectTaggingHandler(rr, req) + + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, http.StatusBadRequest, rr.Body.String()) + } + if !strings.Contains(rr.Body.String(), "InvalidTag") { + t.Fatalf("body = %s, want InvalidTag", rr.Body.String()) + } +} diff --git a/weed/s3api/s3tables/utils.go b/weed/s3api/s3tables/utils.go index d83e498830..8c76b189af 100644 --- a/weed/s3api/s3tables/utils.go +++ b/weed/s3api/s3tables/utils.go @@ -9,6 +9,7 @@ import ( "regexp" "strings" "time" + "unicode/utf8" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" ) @@ -372,17 +373,25 @@ func arnPartitionForRegion(region string) string { // ValidateTags validates tags for S3 Tables. func ValidateTags(tags map[string]string) error { - if len(tags) > 10 { - return fmt.Errorf("validate tags: %d tags more than 10", len(tags)) + return ValidateTagsWithLimit(tags, 10) +} + +// ValidateTagsWithLimit validates a tag set that may hold up to maxTags tags. +func ValidateTagsWithLimit(tags map[string]string, maxTags int) error { + if len(tags) > maxTags { + return fmt.Errorf("validate tags: %d tags more than %d", len(tags), maxTags) } for k, v := range tags { - if len(k) > 128 { + if len(k) == 0 { + return fmt.Errorf("validate tags: empty tag key") + } + if utf8.RuneCountInString(k) > 128 { return fmt.Errorf("validate tags: tag key longer than 128") } if !tagPattern.MatchString(k) { return fmt.Errorf("validate tags key %s error, incorrect key", k) } - if len(v) > 256 { + if utf8.RuneCountInString(v) > 256 { return fmt.Errorf("validate tags: tag value longer than 256") } if !tagPattern.MatchString(v) { diff --git a/weed/s3api/tags.go b/weed/s3api/tags.go index 76830ccf14..49cde79f59 100644 --- a/weed/s3api/tags.go +++ b/weed/s3api/tags.go @@ -63,6 +63,9 @@ func parseTagsHeader(tags string) (map[string]string, error) { if err != nil { return nil, fmt.Errorf("failed to decode tag key '%s': %w", key, err) } + if _, exists := parsedTags[decodedKey]; exists { + return nil, fmt.Errorf("duplicate tag key '%s'", decodedKey) + } if !hasValue { parsedTags[decodedKey] = "" continue @@ -79,3 +82,11 @@ func parseTagsHeader(tags string) (map[string]string, error) { func ValidateTags(tags map[string]string) error { return s3tables.ValidateTags(tags) } + +// maxBucketTags is how many tags a bucket's tag set can hold; an object's +// tag set holds 10. +const maxBucketTags = 50 + +func ValidateBucketTags(tags map[string]string) error { + return s3tables.ValidateTagsWithLimit(tags, maxBucketTags) +} diff --git a/weed/s3api/tags_test.go b/weed/s3api/tags_test.go index a7dd1b806e..f313806ae5 100644 --- a/weed/s3api/tags_test.go +++ b/weed/s3api/tags_test.go @@ -1,6 +1,7 @@ package s3api import ( + "strings" "testing" ) @@ -62,6 +63,18 @@ func TestParseTagsHeader(t *testing.T) { expected: nil, expectError: true, }, + { + name: "duplicate key", + input: "key1=value1&key1=value2", + expected: nil, + expectError: true, + }, + { + name: "encoded duplicate key", + input: "key1=value1&%6bey1=value2", + expected: nil, + expectError: true, + }, { name: "plus signs and equals in values", input: "formula=a%2Bb%3Dc&normal=test", @@ -120,3 +133,19 @@ func TestParseTagsHeader(t *testing.T) { }) } } + +func TestValidateTagsUnicodeLength(t *testing.T) { + // S3 counts tag length in characters, not bytes + multibyteKey := strings.Repeat("中", 128) + multibyteValue := strings.Repeat("中", 256) + + if err := ValidateTags(map[string]string{multibyteKey: multibyteValue}); err != nil { + t.Errorf("Expected 128-char key and 256-char value to pass, got %v", err) + } + if err := ValidateTags(map[string]string{strings.Repeat("中", 129): "v"}); err == nil { + t.Error("Expected 129-char key to fail") + } + if err := ValidateTags(map[string]string{"k": strings.Repeat("中", 257)}); err == nil { + t.Error("Expected 257-char value to fail") + } +} From d6a68468adeacdacc99b5e8ef1e126b351b4257a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:04:05 -0700 Subject: [PATCH 58/70] volume: keep a below-minimum fsync batch open for stragglers (#12016) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * storage: hold a below-minimum fsync batch open for stragglers When the durable-write queue drained, the worker flushed whatever it held, so low-concurrency fsync writes collapsed to one sync per request -- a seek plus a rotation on rotational media no matter the payload. Once the queue empties with the batch still below the configured minimum bytes and requests, the batch now stays open up to SyncBatchOption.Wait, armed once per batch so the added latency stays bounded. DiskLocation carries the option and defaults it; Store exposes a setter the server calls once at startup. * volume server: expose the fsync batch window as -fsync* flags -fsyncMinBatchBytes, -fsyncMinBatchRequests and -fsyncBatchWait carry the operator's batching floor into every disk's SyncBatchOption. -fsyncBatchWait=0 restores the previous flush-on-drain behavior. * server/mini: register the fsync batch window flags weed server and weed mini build their own VolumeServerOptions and left the three new fsync flag pointers nil, so startVolumeServer panicked dereferencing them at startup. * storage: bound the fsync batch window by Wait while the queue refills Once the straggler timer was armed, a steady trickle of queued writes kept taking the len(requests)>0 continue path, so the first write could wait past Wait until the size or count cap flushed the batch. Track the armed deadline and flush once it passes even with requests buffered. * storage: sync the straggler test on the worker taking the first write The fixed sleep let both writes queue together (passing under the old flush-on-drain behavior) or the straggler arrive after the window closed (flaky failure). Poll the drained queue instead so the second write can only land inside the armed window or a later batch. * volume server: default the fsync batch window when its options are unset weed server and weed mini build VolumeServerOptions without the fsync flag pointers, and outside embedders can too. Resolve the batch window from the defaults so startVolumeServer never dereferences nil options. * storage: drive the batch-window test through an injectable timer The window is armed by the worker now, so the straggler provably joins an open batch and expiry is fired by hand instead of racing a sleep. * seaweed-volume: accept -fsyncMinBatch/-fsyncBatchWait flags Same flags and defaults as weed volume so the Rust server's durable write batching can be tuned identically. * seaweed-volume: hold a below-minimum fsync batch open for stragglers Mirrors the Go worker's batchWait: once the queue drains under the minimum batch, the window stays open for stragglers and an expired window still bounds the batch while the queue refills. Paused-time coverage replaces any sleep-based assertion. * seaweed-volume: sync the straggler test on the armed batch window The test fired both writes together, so the straggler could already be queued when the worker drained — a regression that drops mid-window writes from the open batch would still pass. Wait until the worker arms the window before sending the last write (Go's test polls the drained channel for the same guarantee). Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * seaweed-volume: reject fsyncMinBatchBytes that does not fit usize --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- seaweed-volume/Cargo.toml | 2 + seaweed-volume/src/config.rs | 30 ++++ seaweed-volume/src/main.rs | 14 +- seaweed-volume/src/server/grpc_client.rs | 3 + seaweed-volume/src/server/handlers.rs | 1 + seaweed-volume/src/server/profiling.rs | 3 + seaweed-volume/src/server/write_queue.rs | 176 +++++++++++++++++++++-- seaweed-volume/tests/http_integration.rs | 4 +- weed/command/mini.go | 3 + weed/command/server.go | 3 + weed/command/volume.go | 23 +++ weed/server/volume_server.go | 2 + weed/storage/disk_location.go | 2 + weed/storage/store.go | 8 ++ weed/storage/volume_write.go | 67 ++++++++- weed/storage/volume_write_test.go | 84 +++++++++++ 16 files changed, 403 insertions(+), 22 deletions(-) diff --git a/seaweed-volume/Cargo.toml b/seaweed-volume/Cargo.toml index 1e4a58f2a2..29c37d4ff3 100644 --- a/seaweed-volume/Cargo.toml +++ b/seaweed-volume/Cargo.toml @@ -157,6 +157,8 @@ windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [dev-dependencies] tempfile = "3" +# Paused-time tests drive the write-queue batch window deterministically. +tokio = { version = "1", features = ["full", "test-util"] } # Already a transitive dependency of aws-sdk-s3 at a single locked version; # needed directly only for the canned HttpClient in remote_storage::s3 tests. aws-smithy-runtime-api = "1" diff --git a/seaweed-volume/src/config.rs b/seaweed-volume/src/config.rs index a50e52e093..cf73731cc1 100644 --- a/seaweed-volume/src/config.rs +++ b/seaweed-volume/src/config.rs @@ -233,6 +233,25 @@ pub struct Cli { #[arg(long = "volume.allowUntrustedRemoteEndpoints", default_value_t = false)] pub allow_untrusted_remote_endpoints: bool, + /// Minimum bytes an fsync batch gathers before a drained queue flushes + /// without waiting. + #[arg(long = "fsyncMinBatchBytes", default_value_t = 1024 * 1024)] + pub fsync_min_batch_bytes: u64, + + /// Minimum requests an fsync batch gathers before a drained queue flushes + /// without waiting. + #[arg(long = "fsyncMinBatchRequests", default_value_t = 128)] + pub fsync_min_batch_requests: usize, + + /// How long a below-minimum fsync batch waits for more requests once the + /// queue drains; 0 flushes immediately. + #[arg( + long = "fsyncBatchWait", + default_value = "500us", + value_parser = parse_go_duration + )] + pub fsync_batch_wait: std::time::Duration, + /// A file of command line options, each line in optionName=optionValue format. #[arg(long = "options", default_value = "")] pub options: String, @@ -308,6 +327,14 @@ pub struct VolumeServerConfig { /// If true, FetchAndWriteNeedle skips remote S3 endpoint validation /// (allows loopback / link-local / metadata hosts). pub allow_untrusted_remote_endpoints: bool, + /// Minimum bytes an fsync batch gathers before a drained queue flushes + /// without waiting. + pub fsync_min_batch_bytes: u64, + /// Minimum requests an fsync batch gathers before a drained queue flushes + /// without waiting. + pub fsync_min_batch_requests: usize, + /// How long a below-minimum fsync batch waits for stragglers. + pub fsync_batch_wait: std::time::Duration, } pub use crate::storage::needle_map::NeedleMapKind; @@ -938,6 +965,9 @@ fn resolve_config_with_env(cli: Cli, env: EnvLookup<'_>) -> VolumeServerConfig { concurrent_download_limit: cli.concurrent_download_limit_mb as i64 * 1024 * 1024, inflight_upload_data_timeout: cli.inflight_upload_data_timeout, inflight_download_data_timeout: cli.inflight_download_data_timeout, + fsync_min_batch_bytes: cli.fsync_min_batch_bytes, + fsync_min_batch_requests: cli.fsync_min_batch_requests, + fsync_batch_wait: cli.fsync_batch_wait, has_slow_read: cli.has_slow_read, read_buffer_size_mb: cli.read_buffer_size_mb, ldb_timeout: cli.ldb_timeout, diff --git a/seaweed-volume/src/main.rs b/seaweed-volume/src/main.rs index c57a92e3b8..46fe45289d 100644 --- a/seaweed-volume/src/main.rs +++ b/seaweed-volume/src/main.rs @@ -23,7 +23,7 @@ use seaweed_volume::server::request_id::GrpcRequestIdLayer; use seaweed_volume::server::volume_server::{ RuntimeMetricsConfig, VolumeServerState, build_metrics_router, }; -use seaweed_volume::server::write_queue::WriteQueue; +use seaweed_volume::server::write_queue::{self, WriteQueue}; use seaweed_volume::storage::store::Store; use seaweed_volume::storage::types::DiskType; @@ -436,7 +436,17 @@ async fn run( // The write queue is always on: durable (fsync) uploads go through it so // a batch shares one .dat/.idx sync, the way Go's per-volume async writer // group-commits. Non-fsync writes stay on the synchronous path. - let wq = WriteQueue::new(state.clone(), 128); + let min_bytes = usize::try_from(config.fsync_min_batch_bytes) + .map_err(|_| "fsyncMinBatchBytes exceeds this platform's maximum usize value")?; + let wq = WriteQueue::new( + state.clone(), + 128, + write_queue::SyncBatchOption { + min_bytes, + min_requests: config.fsync_min_batch_requests, + wait: config.fsync_batch_wait, + }, + ); let _ = state.write_queue.set(wq); // Set initial metric gauges for concurrent limits and max volumes diff --git a/seaweed-volume/src/server/grpc_client.rs b/seaweed-volume/src/server/grpc_client.rs index 45a0170744..9567e43658 100644 --- a/seaweed-volume/src/server/grpc_client.rs +++ b/seaweed-volume/src/server/grpc_client.rs @@ -393,6 +393,9 @@ mod tests { fix_jpg_orientation: false, read_mode: ReadMode::Local, allow_untrusted_remote_endpoints: false, + fsync_min_batch_bytes: 1024 * 1024, + fsync_min_batch_requests: 128, + fsync_batch_wait: std::time::Duration::from_micros(500), cpu_profile: String::new(), mem_profile: String::new(), compaction_byte_per_second: 0, diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index 6accfed9fe..fdba467c7c 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -7308,6 +7308,7 @@ mod tests { .set(crate::server::write_queue::WriteQueue::new( state.clone(), 128, + crate::server::write_queue::SyncBatchOption::default(), )) .is_ok() ); diff --git a/seaweed-volume/src/server/profiling.rs b/seaweed-volume/src/server/profiling.rs index 9d5398b22c..5622e9d428 100644 --- a/seaweed-volume/src/server/profiling.rs +++ b/seaweed-volume/src/server/profiling.rs @@ -132,6 +132,9 @@ mod tests { fix_jpg_orientation: false, read_mode: ReadMode::Local, allow_untrusted_remote_endpoints: false, + fsync_min_batch_bytes: 1024 * 1024, + fsync_min_batch_requests: 128, + fsync_batch_wait: std::time::Duration::from_micros(500), cpu_profile: String::new(), mem_profile: String::new(), compaction_byte_per_second: 0, diff --git a/seaweed-volume/src/server/write_queue.rs b/seaweed-volume/src/server/write_queue.rs index 01c4f69e72..dc649e0d72 100644 --- a/seaweed-volume/src/server/write_queue.rs +++ b/seaweed-volume/src/server/write_queue.rs @@ -56,6 +56,27 @@ const MAX_CONCURRENT_VOLUME_WRITES: usize = 8; /// retirement is the volume leaving the store. const IDLE_WORKER_POLL: std::time::Duration = std::time::Duration::from_secs(60); +/// Tunes the per-volume durable-write worker. Once the queue drains with the +/// batch still below min_bytes and min_requests, the batch stays open up to +/// wait for stragglers instead of paying an fsync per request. Mirrors Go's +/// storage.SyncBatchOption. +#[derive(Clone, Copy)] +pub struct SyncBatchOption { + pub min_bytes: usize, + pub min_requests: usize, + pub wait: std::time::Duration, +} + +impl Default for SyncBatchOption { + fn default() -> Self { + SyncBatchOption { + min_bytes: 1024 * 1024, + min_requests: MAX_BATCH_SIZE, + wait: std::time::Duration::from_micros(500), + } + } +} + /// Handle for submitting write requests to the per-volume workers. #[derive(Clone)] pub struct WriteQueue { @@ -67,23 +88,35 @@ pub struct WriteQueue { /// alone would multiply the old single queue's capacity by volume count. queue_permits: Arc, workers: Arc>>>, + sync_batch: SyncBatchOption, /// Submissions accepted, for tests asserting which writes route here. #[cfg(test)] submits: Arc, + /// Times a worker armed the straggler window, for tests that must land a + /// write inside an already-open batch. + #[cfg(test)] + windows_armed: Arc, } impl WriteQueue { /// Create a new write queue. Per-volume workers are spawned lazily on the /// first submission for that volume. - pub fn new(state: Arc, capacity: usize) -> Self { + pub fn new( + state: Arc, + capacity: usize, + sync_batch: SyncBatchOption, + ) -> Self { WriteQueue { state, capacity, io_permits: Arc::new(Semaphore::new(MAX_CONCURRENT_VOLUME_WRITES)), queue_permits: Arc::new(Semaphore::new(capacity)), workers: Arc::new(Mutex::new(HashMap::new())), + sync_batch, #[cfg(test)] submits: Arc::new(std::sync::atomic::AtomicU64::new(0)), + #[cfg(test)] + windows_armed: Arc::new(std::sync::atomic::AtomicU64::new(0)), } } @@ -93,6 +126,12 @@ impl WriteQueue { self.submits.load(std::sync::atomic::Ordering::Relaxed) } + /// How many batch windows workers have armed so far. + #[cfg(test)] + fn windows_armed_for_test(&self) -> u64 { + self.windows_armed.load(std::sync::atomic::Ordering::Relaxed) + } + /// Sender for the volume's worker, spawning one on first use. fn worker_for(&self, volume_id: VolumeId) -> mpsc::Sender { let mut workers = self.workers.lock().unwrap(); @@ -107,6 +146,9 @@ impl WriteQueue { state: self.state.clone(), io_permits: self.io_permits.clone(), workers: self.workers.clone(), + sync_batch: self.sync_batch, + #[cfg(test)] + windows_armed: self.windows_armed.clone(), } .run(), ); @@ -191,6 +233,9 @@ struct VolumeWriteWorker { state: Arc, io_permits: Arc, workers: Arc>>>, + sync_batch: SyncBatchOption, + #[cfg(test)] + windows_armed: Arc, } impl VolumeWriteWorker { @@ -238,14 +283,40 @@ impl VolumeWriteWorker { let mut batch = Vec::with_capacity(MAX_BATCH_SIZE); let mut batch_bytes: usize = first.needle.data.len(); batch.push(first); + // Armed once the queue first drains below the minimum batch; holds the + // batch open for stragglers so the wait bound is per batch, not per + // lull (Go's batchWait). + let mut window_deadline = None; while batch.len() < MAX_BATCH_SIZE && batch_bytes < MAX_BATCH_BYTES { - match self.rx.try_recv() { - Ok(req) => { - batch_bytes += req.needle.data.len(); - batch.push(req); - } - Err(_) => break, + if window_deadline.is_some_and(|d| tokio::time::Instant::now() >= d) { + break; } + let req = match self.rx.try_recv() { + Ok(req) => req, + Err(mpsc::error::TryRecvError::Empty) => { + let sync = self.sync_batch; + if sync.wait.is_zero() + || batch_bytes >= sync.min_bytes + || batch.len() >= sync.min_requests + { + // a drained queue flushes right away once it holds enough + break; + } + let deadline = *window_deadline.get_or_insert_with(|| { + #[cfg(test)] + self.windows_armed + .fetch_add(1, std::sync::atomic::Ordering::Relaxed); + tokio::time::Instant::now() + sync.wait + }); + match tokio::time::timeout_at(deadline, self.rx.recv()).await { + Ok(Some(req)) => req, + Ok(None) | Err(_) => break, + } + } + Err(mpsc::error::TryRecvError::Disconnected) => break, + }; + batch_bytes += req.needle.data.len(); + batch.push(req); } debug!("processing write batch of {} requests", batch.len()); @@ -380,7 +451,7 @@ mod tests { async fn test_write_queue_submit_no_volume() { // Submit a write to a non-existent volume -- should return VolumeError::NotFound. let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let needle = Needle { id: 1.into(), @@ -402,7 +473,7 @@ mod tests { async fn test_write_queue_concurrent_submissions() { // Submit multiple concurrent writes -- all should complete (with errors since no volume). let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let mut handles = Vec::new(); for i in 0..10u64 { @@ -430,7 +501,7 @@ mod tests { async fn test_write_queue_batching() { // Verify that many concurrent writes get processed (testing the batching path). let state = make_test_state(); - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); // Submit MAX_BATCH_SIZE requests concurrently let mut handles = Vec::new(); @@ -522,7 +593,7 @@ mod tests { async fn test_write_queue_dropped_sender() { // When the queue is dropped, subsequent submits should fail gracefully. let state = make_test_state(); - let queue = WriteQueue::new(state, 1); + let queue = WriteQueue::new(state, 1, SyncBatchOption::default()); // Clone then drop the original -- the worker keeps running via its rx handle. let queue2 = queue.clone(); @@ -665,7 +736,7 @@ mod tests { .unwrap(); } } - let queue = WriteQueue::new(state, MAX_BATCH_SIZE); + let queue = WriteQueue::new(state, MAX_BATCH_SIZE, SyncBatchOption::default()); let in_append = Arc::new(Barrier::new(2)); let release = Arc::new(Barrier::new(2)); @@ -734,4 +805,85 @@ mod tests { assert!(a1.await.unwrap().is_ok()); assert!(a2.await.unwrap().is_ok()); } + + /// A durable write below the minimum batch rides out its window so a + /// straggler joins the same sync instead of each paying an fsync alone — + /// Go's batchWait. Paused time makes the hold observable: a lone write can + /// only complete once the runtime auto-advances to the window's deadline. + #[tokio::test(start_paused = true)] + async fn test_batch_window_defers_flush_for_stragglers() { + use crate::config::MinFreeSpace; + use crate::storage::types::{DiskType, NeedleId}; + use crate::storage::volume::VolumeSpec; + use std::time::Duration; + + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().to_str().unwrap(); + let state = make_test_state(); + { + let mut store = state.store.write().unwrap(); + store + .add_location( + dir, + dir, + 10, + DiskType::HardDrive, + MinFreeSpace::Percent(1.0), + Vec::new(), + ) + .unwrap(); + store + .add_volume(VolumeId(1), DiskType::HardDrive, &VolumeSpec::default()) + .unwrap(); + } + + let queue = WriteQueue::new( + state, + MAX_BATCH_SIZE, + SyncBatchOption { + min_bytes: usize::MAX, + min_requests: MAX_BATCH_SIZE, + wait: Duration::from_secs(30), + }, + ); + + let needle = |id: u64| Needle { + id: NeedleId(id), + cookie: 0x1111.into(), + data: vec![id as u8; 8], + data_size: 8, + ..Needle::default() + }; + + let start = tokio::time::Instant::now(); + queue.submit(VolumeId(1), needle(1), true).await.unwrap(); + assert!(start.elapsed() >= Duration::from_secs(30)); + + // Wait for the worker to take the second write and open its batch + // window, so the straggler provably lands inside an open batch + // instead of arriving during the initial drain. + let armed = queue.windows_armed_for_test(); + let second = tokio::spawn({ + let queue = queue.clone(); + async move { queue.submit(VolumeId(1), needle(2), true).await } + }); + for _ in 0..10_000 { + if queue.windows_armed_for_test() > armed { + break; + } + tokio::task::yield_now().await; + } + assert!(queue.windows_armed_for_test() > armed); + let third = tokio::spawn({ + let queue = queue.clone(); + async move { queue.submit(VolumeId(1), needle(3), true).await } + }); + let (second, third) = tokio::join!(second, third); + second.unwrap().unwrap(); + third.unwrap().unwrap(); + // One shared window ends both writes ~30s after their batch armed; + // back-to-back windows would keep the third out to ~90s. + let elapsed = start.elapsed(); + assert!(elapsed >= Duration::from_secs(60) && elapsed < Duration::from_secs(90)); + } } diff --git a/seaweed-volume/tests/http_integration.rs b/seaweed-volume/tests/http_integration.rs index 7622fab739..b90234a5ad 100644 --- a/seaweed-volume/tests/http_integration.rs +++ b/seaweed-volume/tests/http_integration.rs @@ -1340,11 +1340,11 @@ async fn non_ascii_fid_and_ttl_are_rejected_not_panicked() { #[tokio::test] async fn write_queue_upload_returns_same_etag_as_direct_write() { - use seaweed_volume::server::write_queue::WriteQueue; + use seaweed_volume::server::write_queue::{SyncBatchOption, WriteQueue}; let (direct_state, _direct_tmp) = test_state(); let (queued_state, _queued_tmp) = test_state(); - let wq = WriteQueue::new(queued_state.clone(), 128); + let wq = WriteQueue::new(queued_state.clone(), 128, SyncBatchOption::default()); let _ = queued_state.write_queue.set(wq); let compressible = "seaweedfs ".repeat(200).into_bytes(); diff --git a/weed/command/mini.go b/weed/command/mini.go index 0d2660438e..7cc848d3b7 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -517,6 +517,9 @@ func initMiniVolumeFlags() { miniOptions.v.hasSlowRead = cmdMini.Flag.Bool("volume.hasSlowRead", true, "if true, prevents slow reads from blocking other requests") miniOptions.v.readBufferSizeMB = cmdMini.Flag.Int("volume.readBufferSizeMB", 4, "read buffer size in MB") miniOptions.v.allowUntrustedRemoteEndpoints = cmdMini.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + miniOptions.v.fsyncMinBatchBytes = cmdMini.Flag.Int64("volume.fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + miniOptions.v.fsyncMinBatchRequests = cmdMini.Flag.Int("volume.fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + miniOptions.v.fsyncBatchWait = cmdMini.Flag.Duration("volume.fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") miniOptions.v.preStopSeconds = cmdMini.Flag.Int("volume.preStopSeconds", 1, "number of seconds between stop send heartbeats and stop volume server (default: 1 for mini)") miniOptions.v.setDiskIOProbeDefaults() } diff --git a/weed/command/server.go b/weed/command/server.go index 5e41c6e3bb..f64f3b8973 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -165,6 +165,9 @@ func init() { serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, " if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.") serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, " larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally") serverOptions.v.allowUntrustedRemoteEndpoints = cmdServer.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + serverOptions.v.fsyncMinBatchBytes = cmdServer.Flag.Int64("volume.fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + serverOptions.v.fsyncMinBatchRequests = cmdServer.Flag.Int("volume.fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + serverOptions.v.fsyncBatchWait = cmdServer.Flag.Duration("volume.fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") serverOptions.v.setDiskIOProbeDefaults() s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") diff --git a/weed/command/volume.go b/weed/command/volume.go index e3244bf473..95b76409c1 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -76,6 +76,9 @@ type VolumeServerOptions struct { readBufferSizeMB *int ldbTimeout *int64 allowUntrustedRemoteEndpoints *bool + fsyncMinBatchBytes *int64 + fsyncMinBatchRequests *int + fsyncBatchWait *time.Duration debug *bool debugPort *int diskIOProbe *bool @@ -147,6 +150,22 @@ func (v *VolumeServerOptions) setDiskIOProbeDefaults() { } } +// syncBatchOption resolves the fsync batch window, defaulting each field so +// commands or embedders that leave the option pointers unset still start. +func (v *VolumeServerOptions) syncBatchOption() storage.SyncBatchOption { + option := storage.DefaultSyncBatchOption + if v.fsyncMinBatchBytes != nil { + option.MinBytes = *v.fsyncMinBatchBytes + } + if v.fsyncMinBatchRequests != nil { + option.MinRequests = *v.fsyncMinBatchRequests + } + if v.fsyncBatchWait != nil { + option.Wait = *v.fsyncBatchWait + } + return option +} + func (v *VolumeServerOptions) applyDiskIOProbeConfig() { v.setDiskIOProbeDefaults() @@ -230,6 +249,9 @@ func init() { v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, " if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.") v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, " larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.") v.allowUntrustedRemoteEndpoints = cmdVolume.Flag.Bool("volume.allowUntrustedRemoteEndpoints", false, "if true, FetchAndWriteNeedle accepts arbitrary remote S3 endpoints including loopback / link-local hosts. Default rejects internal / metadata endpoints.") + v.fsyncMinBatchBytes = cmdVolume.Flag.Int64("fsyncMinBatchBytes", 1*1024*1024, "minimum bytes an fsync batch gathers before a drained queue flushes without waiting") + v.fsyncMinBatchRequests = cmdVolume.Flag.Int("fsyncMinBatchRequests", 128, "minimum requests an fsync batch gathers before a drained queue flushes without waiting") + v.fsyncBatchWait = cmdVolume.Flag.Duration("fsyncBatchWait", 500*time.Microsecond, "how long a below-minimum fsync batch waits for more requests once the queue drains; 0 flushes immediately") v.debug = cmdVolume.Flag.Bool("debug", false, "serves runtime profiling data via pprof on the port specified by -debug.port") v.debugPort = cmdVolume.Flag.Int("debug.port", 6060, "http port for debugging") v.setDiskIOProbeDefaults() @@ -460,6 +482,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v *v.ldbTimeout, *v.allowUntrustedRemoteEndpoints, diskProbeConfig, + v.syncBatchOption(), ) // starting grpc server grpcS := v.startGrpcService(volumeServer) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index ff1c85d044..93351eca44 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -93,6 +93,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, ldbTimeout int64, allowUntrustedRemoteEndpoints bool, diskProbeConfig stats.DiskIOProbeConfig, + syncBatch storage.SyncBatchOption, ) *VolumeServer { v := util.GetViper() @@ -149,6 +150,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, vs.store = storage.NewStore(vs.grpcDialOption, ip, port, grpcPort, publicUrl, id, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes, diskTags, ldbTimeout, diskProbeConfig) vs.store.MasterFn = vs.getCurrentMaster + vs.store.SetSyncBatchOption(syncBatch) vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) handleStaticResources(adminMux) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index ce76412a76..e904bae7dc 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -58,6 +58,7 @@ type DiskLocation struct { isDiskSpaceLow atomic.Bool isDiskUnavailable atomic.Bool + syncBatch SyncBatchOption closeCh chan struct{} } @@ -119,6 +120,7 @@ func NewDiskLocation(dir string, maxVolumeCount int32, minFreeSpace util.MinFree MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, MinFreeSpace: minFreeSpace, + syncBatch: DefaultSyncBatchOption, } location.volumes = make(map[needle.VolumeId]*Volume) location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) diff --git a/weed/storage/store.go b/weed/storage/store.go index 0d2b51b50d..d213ee6f9e 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -1359,6 +1359,14 @@ func (s *Store) GetVolumeSizeLimit() uint64 { return s.volumeSizeLimit.Load() } +// SetSyncBatchOption applies the durable-write batch window to every disk. +// It runs at startup, before any volume takes a write. +func (s *Store) SetSyncBatchOption(syncBatch SyncBatchOption) { + for _, location := range s.Locations { + location.syncBatch = syncBatch + } +} + func (s *Store) SetPreallocate(x bool) { s.preallocate.Store(x) } diff --git a/weed/storage/volume_write.go b/weed/storage/volume_write.go index 88072c9aa3..f119425fff 100644 --- a/weed/storage/volume_write.go +++ b/weed/storage/volume_write.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "os" + "time" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/storage/backend" @@ -661,6 +662,29 @@ func (v *Volume) processBatchRun(run []*needle.AsyncRequest) { } } +const ( + syncBatchMaxBytes = 4 * 1024 * 1024 + syncBatchMaxRequests = 128 +) + +// SyncBatchOption tunes the volume's durable-write worker. Once the queue +// drains with the batch still below MinBytes and MinRequests, the batch stays +// open up to Wait for stragglers instead of paying an fsync per request. +type SyncBatchOption struct { + MinBytes int64 + MinRequests int + Wait time.Duration +} + +var DefaultSyncBatchOption = SyncBatchOption{ + MinBytes: 1 * 1024 * 1024, + MinRequests: syncBatchMaxRequests, + Wait: 500 * time.Microsecond, +} + +// syncBatchAfter arms the batch window; a var so tests can drive it. +var syncBatchAfter = time.After + // startWorker returns the volume's batch-write channel, creating it and its // goroutine on first use, and nil once stopWorker has run. func (v *Volume) startWorker() chan *needle.AsyncRequest { @@ -672,8 +696,12 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { if v.asyncRequestsChan != nil { return v.asyncRequestsChan } - requests := make(chan *needle.AsyncRequest, 128) + requests := make(chan *needle.AsyncRequest, syncBatchMaxRequests) v.asyncRequestsChan = requests + syncBatch := DefaultSyncBatchOption + if v.location != nil { + syncBatch = v.location.syncBatch + } go func() { chanClosed := false for { @@ -681,15 +709,25 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { if chanClosed { break } - currentRequests := make([]*needle.AsyncRequest, 0, 128) + currentRequests := make([]*needle.AsyncRequest, 0, syncBatchMaxRequests) currentBytesToWrite := int64(0) + var batchDeadline time.Time + var batchWait <-chan time.Time for { - request, ok := <-requests + var request *needle.AsyncRequest + queueOpen := true + select { + case request, queueOpen = <-requests: + case <-batchWait: + } // volume may be closed - if !ok { + if !queueOpen { chanClosed = true break } + if request == nil { + break + } if MaxPossibleVolumeSize < v.ContentSize()+uint64(currentBytesToWrite+request.ActualSize) { request.Complete(0, 0, false, fmt.Errorf("volume size limit %d exceeded! current size is %d", MaxPossibleVolumeSize, v.ContentSize())) @@ -698,10 +736,27 @@ func (v *Volume) startWorker() chan *needle.AsyncRequest { currentRequests = append(currentRequests, request) currentBytesToWrite += request.ActualSize // submit at most 4M bytes or 128 requests at one time to decrease request delay. - // it also need to break if there is no data in channel to avoid io hang. - if currentBytesToWrite >= 4*1024*1024 || len(currentRequests) >= 128 || len(requests) == 0 { + if currentBytesToWrite >= syncBatchMaxBytes || len(currentRequests) >= syncBatchMaxRequests { break } + // a refilled queue must not outwait the armed deadline + if !batchDeadline.IsZero() && !time.Now().Before(batchDeadline) { + break + } + if len(requests) > 0 { + continue + } + if syncBatch.Wait <= 0 || currentBytesToWrite >= syncBatch.MinBytes || len(currentRequests) >= syncBatch.MinRequests { + // a drained queue flushes right away once it holds enough + break + } + // flushing now would sync a singleton batch; hold the window + // open for stragglers, armed once per batch so the added + // latency stays bounded by Wait + if batchWait == nil { + batchDeadline = time.Now().Add(syncBatch.Wait) + batchWait = syncBatchAfter(syncBatch.Wait) + } } if len(currentRequests) == 0 { continue diff --git a/weed/storage/volume_write_test.go b/weed/storage/volume_write_test.go index d35cdda2b2..2fa479067b 100644 --- a/weed/storage/volume_write_test.go +++ b/weed/storage/volume_write_test.go @@ -6,12 +6,14 @@ import ( "fmt" "math" "os" + "sync/atomic" "testing" "time" "github.com/stretchr/testify/assert" "github.com/seaweedfs/seaweedfs/weed/stats" + "github.com/seaweedfs/seaweedfs/weed/storage/backend" "github.com/seaweedfs/seaweedfs/weed/storage/needle" "github.com/seaweedfs/seaweedfs/weed/storage/super_block" "github.com/seaweedfs/seaweedfs/weed/storage/types" @@ -497,3 +499,85 @@ func TestDoWriteRequestRefusesReadOnlyBeforeAppend(t *testing.T) { t.Fatalf("idx size changed; needle must not index") } } + +type syncCountingBackend struct { + backend.BackendStorageFile + syncs atomic.Int64 +} + +func (b *syncCountingBackend) Sync() error { + b.syncs.Add(1) + return b.BackendStorageFile.Sync() +} + +// A drained queue used to flush whatever it held, turning durable writes into +// one fsync per request at low concurrency. With a batch window configured, a +// straggler that lands inside the window joins the same sync. +func TestSyncBatchDefersFlushForStragglers(t *testing.T) { + dir := t.TempDir() + + v, err := NewVolume(dir, dir, "", 7, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0) + if err != nil { + t.Fatalf("volume creation: %v", err) + } + defer v.Close() + + counting := &syncCountingBackend{BackendStorageFile: v.DataBackend} + v.DataBackend = counting + writeBytes := int64(needle.GetActualSize(16, v.Version())) + v.location = &DiskLocation{syncBatch: SyncBatchOption{ + MinBytes: writeBytes * 2, + MinRequests: syncBatchMaxRequests, + Wait: time.Minute, + }} + + armed := make(chan chan time.Time, 4) + prevAfter := syncBatchAfter + syncBatchAfter = func(time.Duration) <-chan time.Time { + window := make(chan time.Time, 1) + armed <- window + return window + } + defer func() { syncBatchAfter = prevAfter }() + + done := make(chan error, 3) + writeAsync := func(id uint64) { + n := new(needle.Needle) + n.Data = make([]byte, 16) + n.Checksum = needle.NewCRC(n.Data) + n.Id = types.Uint64ToNeedleId(id) + _, _, _, writeErr := v.writeNeedle2(n, true, true, false) + done <- writeErr + } + windowFor := func(id uint64) chan time.Time { + go writeAsync(id) + window := <-armed + select { + case <-done: + t.Fatalf("write %d flushed while its batch window was still open", id) + default: + } + return window + } + + windowFor(1) + // the straggler fills the batch to MinBytes, closing the window early + go writeAsync(2) + for i := 0; i < 2; i++ { + if writeErr := <-done; writeErr != nil { + t.Fatalf("write needle: %v", writeErr) + } + } + if syncs := counting.syncs.Load(); syncs != 1 { + t.Fatalf("expected both writes to share one .dat sync, got %d", syncs) + } + + window := windowFor(3) + window <- time.Now() + if writeErr := <-done; writeErr != nil { + t.Fatalf("write needle: %v", writeErr) + } + if syncs := counting.syncs.Load(); syncs != 2 { + t.Fatalf("expected a lone write to take its own sync, got %d total", syncs) + } +} From 9e7fd2eb90f946e451c08626c970cd7665282afd Mon Sep 17 00:00:00 2001 From: Jay Hemnani <193022578+jayhemnani9910@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:34:41 +0530 Subject: [PATCH 59/70] filer: match If-Match and If-None-Match against "*" and ETag lists (#12026) * filer: match If-Match and If-None-Match against "*" and ETag lists checkPreconditions compared the whole header as one ETag, so "*", a list of ETags and a weak W/ tag in If-None-Match never matched (RFC 9110 13.1.1, 13.1.2). * filer: split etag lists only outside quoted tags A comma inside a quoted opaque tag is a legal etagc, not a list separator; strings.Split cut such tags into fragments that CanonicalizeETag could turn into a false match. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer: match repeated If-Match and If-None-Match header lines Header.Get reads only the first field line, so a match on a later repeated header was missed; join all field values like any other comma-separated list. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer: ignore empty elements in etag precondition lists * filer: let a present-but-empty precondition line suppress the date check --------- Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf --- weed/server/filer_server_handlers_read.go | 58 ++++++++- ..._server_handlers_read_precondition_test.go | 115 ++++++++++++++++++ 2 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 weed/server/filer_server_handlers_read_precondition_test.go diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 2fa32fa2ff..cf1e2391e2 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -50,10 +50,10 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent } w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat)) - ifMatchETagHeader := r.Header.Get("If-Match") + ifMatchValues := r.Header.Values("If-Match") ifUnmodifiedSinceHeader := r.Header.Get("If-Unmodified-Since") - if ifMatchETagHeader != "" { - if util.CanonicalizeETag(etag) != util.CanonicalizeETag(ifMatchETagHeader) { + if len(ifMatchValues) != 0 { + if !etagListMatches(strings.Join(ifMatchValues, ","), etag, false) { w.WriteHeader(http.StatusPreconditionFailed) return true } @@ -66,10 +66,10 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent } } - ifNoneMatchETagHeader := r.Header.Get("If-None-Match") + ifNoneMatchValues := r.Header.Values("If-None-Match") ifModifiedSinceHeader := r.Header.Get("If-Modified-Since") - if ifNoneMatchETagHeader != "" { - if util.CanonicalizeETag(etag) == util.CanonicalizeETag(ifNoneMatchETagHeader) { + if len(ifNoneMatchValues) != 0 { + if etagListMatches(strings.Join(ifNoneMatchValues, ","), etag, true) { SetEtag(w, etag) w.WriteHeader(http.StatusNotModified) return true @@ -87,6 +87,52 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, entry *filer.Ent return false } +// etagListMatches reports whether an If-Match or If-None-Match value, "*" or a +// list of entity tags, matches etag (RFC 9110 13.1.1, 13.1.2). If-Match uses the +// strong comparison, so a weak tag never matches; If-None-Match the weak one. +func etagListMatches(header, etag string, weak bool) bool { + if strings.TrimSpace(header) == "*" { + return true + } + for _, tag := range splitETagList(header) { + tag = strings.TrimSpace(tag) + if strings.HasPrefix(tag, "W/") { + if !weak { + continue + } + tag = tag[len("W/"):] + } + // Empty list elements are ignored, so padding like a trailing + // comma cannot match an empty opaque tag. + if tag == "" { + continue + } + if util.CanonicalizeETag(tag) == util.CanonicalizeETag(etag) { + return true + } + } + return false +} + +// splitETagList splits an entity-tag list on the commas between tags; a comma +// inside a quoted opaque tag does not split. +func splitETagList(header string) []string { + var tags []string + start, quoted := 0, false + for i := 0; i < len(header); i++ { + switch header[i] { + case '"': + quoted = !quoted + case ',': + if !quoted { + tags = append(tags, header[start:i]) + start = i + 1 + } + } + } + return append(tags, header[start:]) +} + func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { ctx := r.Context() path := r.URL.Path diff --git a/weed/server/filer_server_handlers_read_precondition_test.go b/weed/server/filer_server_handlers_read_precondition_test.go new file mode 100644 index 0000000000..053056a130 --- /dev/null +++ b/weed/server/filer_server_handlers_read_precondition_test.go @@ -0,0 +1,115 @@ +package weed_server + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" +) + +func TestCheckPreconditionsETagLists(t *testing.T) { + entry := &filer.Entry{Attr: filer.Attr{Mtime: time.Unix(1700000000, 0), Md5: []byte{0xb1, 0x94}}} + const etag = `"b194"` + + tests := []struct { + header string + values []string + want int + }{ + {"If-None-Match", []string{etag}, http.StatusNotModified}, + {"If-None-Match", []string{`*`}, http.StatusNotModified}, + {"If-None-Match", []string{`W/` + etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x", ` + etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x"`, etag}, http.StatusNotModified}, + {"If-None-Match", []string{`"x"`}, http.StatusOK}, + {"If-None-Match", []string{`"b194,old"`}, http.StatusOK}, + {"If-Match", []string{etag}, http.StatusOK}, + {"If-Match", []string{`*`}, http.StatusOK}, + {"If-Match", []string{`"x", ` + etag}, http.StatusOK}, + {"If-Match", []string{`"x"`, etag}, http.StatusOK}, + {"If-Match", []string{`W/` + etag}, http.StatusPreconditionFailed}, + {"If-Match", []string{`"x"`}, http.StatusPreconditionFailed}, + {"If-Match", []string{`"b194,old"`}, http.StatusPreconditionFailed}, + } + for _, tt := range tests { + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + for _, v := range tt.values { + r.Header.Add(tt.header, v) + } + w := httptest.NewRecorder() + got := http.StatusOK + if checkPreconditions(w, r, entry) { + got = w.Code + } + if got != tt.want { + t.Errorf("%s: %q -> %d, want %d", tt.header, tt.values, got, tt.want) + } + } +} + +func TestCheckPreconditionsPresentButEmptyHeaders(t *testing.T) { + entry := &filer.Entry{Attr: filer.Attr{Mtime: time.Unix(1700000000, 0), Md5: []byte{0xb1, 0x94}}} + before := time.Unix(1700000000-60, 0).UTC().Format(http.TimeFormat) + after := time.Unix(1700000000+60, 0).UTC().Format(http.TimeFormat) + + // An empty If-Match line is still an If-Match: it matches nothing, and + // If-Unmodified-Since stays ignored. + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-Match", "") + r.Header.Set("If-Unmodified-Since", after) + if w := httptest.NewRecorder(); !checkPreconditions(w, r, entry) || w.Code != http.StatusPreconditionFailed { + t.Errorf("empty If-Match -> %d, want 412", w.Code) + } + + // An empty If-None-Match line matches nothing and suppresses + // If-Modified-Since, so the request proceeds. + r = httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-None-Match", "") + r.Header.Set("If-Modified-Since", after) + if checkPreconditions(httptest.NewRecorder(), r, entry) { + t.Error("empty If-None-Match with matching If-Modified-Since -> 304, want 200") + } + + r = httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + r.Header.Set("If-Modified-Since", before) + if checkPreconditions(httptest.NewRecorder(), r, entry) { + t.Error("old If-Modified-Since alone -> 304, want 200") + } +} + +func TestCheckPreconditionsIgnoresEmptyETagListElements(t *testing.T) { + entry := &filer.Entry{ + Attr: filer.Attr{Mtime: time.Unix(1700000000, 0)}, + Remote: &filer_pb.RemoteEntry{RemoteSize: 1}, + } + + tests := []struct { + header string + values []string + want int + }{ + {"If-None-Match", []string{`"x",`}, http.StatusOK}, + {"If-None-Match", []string{`"x",,"y"`}, http.StatusOK}, + {"If-None-Match", []string{`""`}, http.StatusNotModified}, + {"If-None-Match", []string{`W/""`}, http.StatusNotModified}, + {"If-Match", []string{`"x",`}, http.StatusPreconditionFailed}, + {"If-Match", []string{`""`}, http.StatusOK}, + } + for _, tt := range tests { + r := httptest.NewRequest(http.MethodGet, "/d/f.txt", nil) + for _, v := range tt.values { + r.Header.Add(tt.header, v) + } + w := httptest.NewRecorder() + got := http.StatusOK + if checkPreconditions(w, r, entry) { + got = w.Code + } + if got != tt.want { + t.Errorf("%s: %q -> %d, want %d", tt.header, tt.values, got, tt.want) + } + } +} From 59cccc788c25c905e660fb745d9f1d31cd2e89be Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:04:45 -0700 Subject: [PATCH 60/70] master: read the writable-volumes cap from master.toml (#12009) * master: read the writable-volumes cap from master.toml -maxWritableVolumes becomes the master.max_writable_volumes config key, so the new master limit is set the same way as other master options instead of a flag wired into every command that embeds a master. * master: name the writable cap master.max_writable_volumes_per_layout The cap applies per volume layout (collection + replica placement + ttl + disk type), so the key names its scope instead of reading as a global cap. * master: keep an embedder's MaxWritableVolumes when the key is unset NewMasterServer overwrote option.MaxWritableVolumes with the viper value, which reads as 0 whenever master.toml omits the key, silently dropping a cap a Go caller supplied through MasterOption. Seed the default from the option instead so the field survives an absent key; an explicit config entry still wins. * master: drop stray nested-checkout gitlinks Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/master.go | 3 --- weed/command/master_follower.go | 1 - weed/command/mini.go | 1 - weed/command/scaffold/master.toml | 7 +++++++ weed/command/server.go | 1 - weed/server/master_server.go | 2 ++ 6 files changed, 9 insertions(+), 6 deletions(-) diff --git a/weed/command/master.go b/weed/command/master.go index d205c3a6a9..64b9883bfa 100644 --- a/weed/command/master.go +++ b/weed/command/master.go @@ -57,7 +57,6 @@ type MasterOptions struct { fileSizeLimitMB *int volumePreallocate *bool maxParallelVacuumPerServer *int - maxWritableVolumes *int vacuumIntervalSeconds *int vacuumDeleteEmptyAfterSeconds *int // pulseSeconds *int @@ -96,7 +95,6 @@ func init() { m.fileSizeLimitMB = cmdMaster.Flag.Int("fileSizeLimitMB", 256, "limit the file size accepted by /submit, should match the volume servers' -fileSizeLimitMB (-volume.fileSizeLimitMB under weed server or weed mini, which set this for you)") m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.") m.maxParallelVacuumPerServer = cmdMaster.Flag.Int("maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel per volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") - m.maxWritableVolumes = cmdMaster.Flag.Int("maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") m.vacuumIntervalSeconds = cmdMaster.Flag.Int("vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") m.vacuumDeleteEmptyAfterSeconds = cmdMaster.Flag.Int("vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") // m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats") @@ -479,7 +477,6 @@ func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOp FileSizeLimitMB: *m.fileSizeLimitMB, VolumePreallocate: *m.volumePreallocate, MaxParallelVacuumPerServer: *m.maxParallelVacuumPerServer, - MaxWritableVolumes: *m.maxWritableVolumes, VacuumIntervalSeconds: *m.vacuumIntervalSeconds, VacuumDeleteEmptyAfterSeconds: *m.vacuumDeleteEmptyAfterSeconds, // PulseSeconds: *m.pulseSeconds, diff --git a/weed/command/master_follower.go b/weed/command/master_follower.go index 95c065775e..4e0983eb96 100644 --- a/weed/command/master_follower.go +++ b/weed/command/master_follower.go @@ -47,7 +47,6 @@ func init() { mf.metricsIntervalSec = aws.Int(0) mf.raftResumeState = aws.Bool(false) mf.maxParallelVacuumPerServer = aws.Int(1) - mf.maxWritableVolumes = aws.Int(0) mf.vacuumIntervalSeconds = aws.Int(840) mf.vacuumDeleteEmptyAfterSeconds = aws.Int(0) mf.telemetryUrl = aws.String("https://telemetry.seaweedfs.com/api/collect") diff --git a/weed/command/mini.go b/weed/command/mini.go index 7cc848d3b7..05a2c1f752 100644 --- a/weed/command/mini.go +++ b/weed/command/mini.go @@ -446,7 +446,6 @@ func initMiniMasterFlags() { miniMasterOptions.volumeSizeLimitMB = cmdMini.Flag.Uint("master.volumeSizeLimitMB", defaultMiniVolumeSizeMB, "Master stops directing writes to oversized volumes (default: 128MB for mini)") miniMasterOptions.volumePreallocate = cmdMini.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") miniMasterOptions.maxParallelVacuumPerServer = cmdMini.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") - miniMasterOptions.maxWritableVolumes = cmdMini.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") miniMasterOptions.vacuumIntervalSeconds = cmdMini.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") miniMasterOptions.vacuumDeleteEmptyAfterSeconds = cmdMini.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") miniMasterOptions.defaultReplication = cmdMini.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") diff --git a/weed/command/scaffold/master.toml b/weed/command/scaffold/master.toml index 3e06fa2259..dfa8472030 100644 --- a/weed/command/scaffold/master.toml +++ b/weed/command/scaffold/master.toml @@ -4,6 +4,13 @@ # /etc/seaweedfs/master.toml # this file is read by master +[master] +# cap on writable volumes per volume layout (collection + replica placement + +# ttl + disk type) that write assignments spread over; fullest picked first. +# 0 = no limit +max_writable_volumes_per_layout = 0 + + [master.maintenance] # periodically run these scripts are the same as running them from 'weed shell' # Scripts are skipped while an admin server is connected. diff --git a/weed/command/server.go b/weed/command/server.go index f64f3b8973..a1d17a5fab 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -98,7 +98,6 @@ func init() { masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", util.DefaultVolumeSizeLimitMB, "Master stops directing writes to oversized volumes.") masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.") masterOptions.maxParallelVacuumPerServer = cmdServer.Flag.Int("master.maxParallelVacuumPerServer", 1, "maximum number of volumes to vacuum in parallel on one volume server; 0 for no limit (deprecated, use master.vacuum.max_parallel_per_server in master.toml)") - masterOptions.maxWritableVolumes = cmdServer.Flag.Int("master.maxWritableVolumes", 0, "maximum writable volumes in a volume layout (collection + replica placement + ttl + disk type) assignments are spread over, fullest picked first; 0 means no limit") masterOptions.vacuumIntervalSeconds = cmdServer.Flag.Int("master.vacuumIntervalSeconds", 840, "seconds between automatic vacuum sweeps") masterOptions.vacuumDeleteEmptyAfterSeconds = cmdServer.Flag.Int("master.vacuumDeleteEmptyAfterSeconds", 0, "automatic sweep deletes volume copies that stay empty this many seconds; 0 disables") masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.") diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 1e6cb58df9..595259c7b1 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -134,6 +134,8 @@ func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.Se if v.IsSet("master.vacuum.max_parallel_per_server") { option.MaxParallelVacuumPerServer = v.GetInt("master.vacuum.max_parallel_per_server") } + v.SetDefault("master.max_writable_volumes_per_layout", option.MaxWritableVolumes) + option.MaxWritableVolumes = v.GetInt("master.max_writable_volumes_per_layout") topology.VolumeGrowStrategy.Copy1Count = v.GetUint32("master.volume_growth.copy_1") topology.VolumeGrowStrategy.Copy2Count = v.GetUint32("master.volume_growth.copy_2") From 8841eb6c7044ccfc334dbcae0eee1f5601b95dad Mon Sep 17 00:00:00 2001 From: Jay Hemnani <193022578+jayhemnani9910@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:35:29 +0530 Subject: [PATCH 61/70] volume: check If-None-Match before If-Modified-Since and match ETag lists (#12027) * filer: match If-Match and If-None-Match against "*" and ETag lists checkPreconditions compared the whole header as one ETag, so "*", a list of ETags and a weak W/ tag in If-None-Match never matched (RFC 9110 13.1.1, 13.1.2). * volume: check If-None-Match before If-Modified-Since and match ETag lists A volume GET answered 304 from If-Modified-Since even when If-None-Match named a different ETag, and compared If-None-Match as one exact string, so "*", lists and weak tags never matched (RFC 9110 13.1.2, 13.2.2). * filer: split etag lists only outside quoted tags A comma inside a quoted opaque tag is a legal etagc, not a list separator; strings.Split cut such tags into fragments that CanonicalizeETag could turn into a false match. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * filer: match repeated If-Match and If-None-Match header lines Header.Get reads only the first field line, so a match on a later repeated header was missed; join all field values like any other comma-separated list. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: match repeated If-None-Match header lines Header.Get reads only the first field line; join all field values so a match on a later repeated header is seen. Also covers a comma inside a quoted opaque tag. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: send the etag on a 304 like the filer does RFC 9110 15.4.5 lists ETag among the fields a 304 must carry when a 200 would send it; SetEtag now runs before the notModified check. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: mirror the conditional-GET fixes in the rust server not_modified_response now checks If-None-Match before If-Modified-Since, matches "*", weak tags, quoted commas and repeated field lines through the same list rules, and sends the etag on a 304. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * volume: treat an empty If-None-Match like the Go server does * filer: ignore empty elements in etag precondition lists * volume: ignore empty elements in etag lists like the filer * filer: let a present-but-empty precondition line suppress the date check * volume: gate If-Modified-Since on a present If-None-Match line * volume: gate If-Modified-Since on header presence like the Go server --------- Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: chrislusf --- seaweed-volume/src/server/handlers.rs | 157 +++++++++++++++--- weed/server/volume_server_handlers_read.go | 31 ++-- .../volume_server_handlers_read_test.go | 37 +++++ 3 files changed, 187 insertions(+), 38 deletions(-) diff --git a/seaweed-volume/src/server/handlers.rs b/seaweed-volume/src/server/handlers.rs index fdba467c7c..1310793564 100644 --- a/seaweed-volume/src/server/handlers.rs +++ b/seaweed-volume/src/server/handlers.rs @@ -1519,41 +1519,78 @@ fn not_modified_response( etag: &str, last_modified_str: &Option, ) -> Option { - // If-Modified-Since first (Go order), then If-None-Match. + let not_modified = || { + let mut resp = StatusCode::NOT_MODIFIED.into_response(); + resp.headers_mut() + .insert(header::ETAG, etag.parse().unwrap()); + if let Some(lm) = last_modified_str { + resp.headers_mut() + .insert(header::LAST_MODIFIED, lm.parse().unwrap()); + } + resp + }; + + // If-Modified-Since applies only when there is no If-None-Match (RFC 9110 13.2.2). + if headers.contains_key(header::IF_NONE_MATCH) { + let inm = headers + .get_all(header::IF_NONE_MATCH) + .iter() + .map(|v| v.to_str().unwrap_or("")) + .collect::>() + .join(","); + return etag_list_matches(&inm, etag).then(not_modified); + } + if n.last_modified > 0 - && let Some(ims_header) = headers.get(header::IF_MODIFIED_SINCE) - && let Ok(ims_str) = ims_header.to_str() + && let Some(ims_str) = headers + .get(header::IF_MODIFIED_SINCE) + .and_then(|v| v.to_str().ok()) { // Parse HTTP date format: "Mon, 02 Jan 2006 15:04:05 GMT" if let Ok(ims_time) = chrono::NaiveDateTime::parse_from_str(ims_str, "%a, %d %b %Y %H:%M:%S GMT") && (n.last_modified as i64) <= ims_time.and_utc().timestamp() { - let mut resp = StatusCode::NOT_MODIFIED.into_response(); - if let Some(lm) = last_modified_str { - resp.headers_mut() - .insert(header::LAST_MODIFIED, lm.parse().unwrap()); - } - // Go sets ETag after the 304 paths, so a 304 has no ETag. - return Some(resp); + return Some(not_modified()); } } - - if let Some(if_none_match) = headers.get(header::IF_NONE_MATCH) - && let Ok(inm) = if_none_match.to_str() - && inm == etag - { - let mut resp = StatusCode::NOT_MODIFIED.into_response(); - if let Some(lm) = last_modified_str { - resp.headers_mut() - .insert(header::LAST_MODIFIED, lm.parse().unwrap()); - } - // Go sets ETag after the 304 paths, so a 304 has no ETag. - return Some(resp); - } None } +/// Weak comparison against "*" or a comma-separated entity-tag list (RFC 9110 13.1.2). +fn etag_list_matches(header: &str, etag: &str) -> bool { + if header.trim() == "*" { + return true; + } + let etag = etag.trim_matches('"'); + split_etag_list(header).into_iter().any(|tag| { + let tag = tag.trim(); + let tag = tag.strip_prefix("W/").unwrap_or(tag); + // Empty list elements are ignored, so padding like a trailing + // comma cannot match an empty opaque tag. + !tag.is_empty() && tag.trim_matches('"') == etag + }) +} + +/// Splits an entity-tag list on the commas between tags; a comma inside a +/// quoted opaque tag does not split. +fn split_etag_list(header: &str) -> Vec<&str> { + let mut tags = Vec::new(); + let (mut start, mut quoted) = (0, false); + for (i, b) in header.bytes().enumerate() { + match b { + b'"' => quoted = !quoted, + b',' if !quoted => { + tags.push(&header[start..i]); + start = i + 1; + } + _ => {} + } + } + tags.push(&header[start..]); + tags +} + /// The 200 reply headers, and the extension, which falls back to the stored name's. fn read_response_headers( n: &Needle, @@ -5644,6 +5681,80 @@ mod tests { delete_handler(State(state.clone()), request).await.status() } + #[test] + fn test_not_modified_checks_if_none_match_first() { + let n = Needle { + last_modified: 1700000000, + ..Needle::default() + }; + let (etag, last_modified_str) = etag_and_last_modified(&n); + let bare = n.etag(); + let before = "Tue, 14 Nov 2023 22:12:20 GMT".to_string(); + let after = "Tue, 14 Nov 2023 22:14:20 GMT".to_string(); + + let cases: Vec<(Vec<(axum::http::HeaderName, String)>, bool)> = vec![ + (vec![(header::IF_NONE_MATCH, format!("\"{bare}\""))], true), + (vec![(header::IF_NONE_MATCH, format!("W/\"{bare}\""))], true), + ( + vec![(header::IF_NONE_MATCH, format!("\"x\", \"{bare}\""))], + true, + ), + (vec![(header::IF_NONE_MATCH, "*".into())], true), + (vec![(header::IF_NONE_MATCH, "\"x\"".into())], false), + ( + vec![ + (header::IF_NONE_MATCH, "\"x\"".into()), + (header::IF_NONE_MATCH, format!("\"{bare}\"")), + ], + true, + ), + ( + vec![(header::IF_NONE_MATCH, format!("\"{bare},old\""))], + false, + ), + ( + vec![(header::IF_NONE_MATCH, format!("\"x\",,\"{bare}\""))], + true, + ), + ( + vec![ + (header::IF_NONE_MATCH, "\"x\"".into()), + (header::IF_MODIFIED_SINCE, after.clone()), + ], + false, + ), + ( + vec![ + (header::IF_NONE_MATCH, "".into()), + (header::IF_MODIFIED_SINCE, after.clone()), + ], + false, + ), + (vec![(header::IF_MODIFIED_SINCE, after)], true), + (vec![(header::IF_MODIFIED_SINCE, before)], false), + ]; + for (pairs, want) in cases { + let mut headers = HeaderMap::new(); + for (name, value) in &pairs { + headers.append(name, value.parse().unwrap()); + } + assert_eq!( + not_modified_response(&n, &headers, &etag, &last_modified_str).is_some(), + want, + "{pairs:?}" + ); + } + + let mut headers = HeaderMap::new(); + headers.insert( + header::IF_NONE_MATCH, + format!("\"{bare}\"").parse().unwrap(), + ); + let resp = not_modified_response(&n, &headers, &etag, &last_modified_str).unwrap(); + assert!(resp.headers().contains_key(header::ETAG)); + assert!(resp.headers().contains_key(header::LAST_MODIFIED)); + } + /// A GET whose needle read is parked must not hold the store lock: a /// writer, here a real append to the same volume, must get through. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 0c46428920..16ab5bf96a 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -149,6 +149,20 @@ func (vs *VolumeServer) proxyReqToTargetServer(w http.ResponseWriter, r *http.Re } } +// notModified checks If-Modified-Since only when there is no If-None-Match +// (RFC 9110 13.2.2). +func notModified(r *http.Request, etag string, lastModified uint64) bool { + if inm := r.Header.Values("If-None-Match"); len(inm) != 0 { + return etagListMatches(strings.Join(inm, ","), etag, true) + } + ims := r.Header.Get("If-Modified-Since") + if ims == "" || lastModified == 0 { + return false + } + t, err := time.Parse(http.TimeFormat, ims) + return err == nil && t.Unix() >= int64(lastModified) +} + func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) { n := new(needle.Needle) vid, fid, filename, ext, _ := parseURLPath(r.URL.Path) @@ -244,21 +258,9 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } if n.LastModified != 0 { w.Header().Set("Last-Modified", time.Unix(int64(n.LastModified), 0).UTC().Format(http.TimeFormat)) - if r.Header.Get("If-Modified-Since") != "" { - if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil { - if t.Unix() >= int64(n.LastModified) { - if readOption.IsMetaOnly { - if v := vs.store.GetVolume(volumeId); v != nil { - v.NoteMetaOnlySuccess() - } - } - w.WriteHeader(http.StatusNotModified) - return - } - } - } } - if inm := r.Header.Get("If-None-Match"); inm == "\""+n.Etag()+"\"" { + SetEtag(w, n.Etag()) + if notModified(r, n.Etag(), n.LastModified) { if readOption.IsMetaOnly { if v := vs.store.GetVolume(volumeId); v != nil { v.NoteMetaOnlySuccess() @@ -267,7 +269,6 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotModified) return } - SetEtag(w, n.Etag()) if n.HasPairs() { pairMap := make(map[string]string) diff --git a/weed/server/volume_server_handlers_read_test.go b/weed/server/volume_server_handlers_read_test.go index 758ff52ea8..1402a9954d 100644 --- a/weed/server/volume_server_handlers_read_test.go +++ b/weed/server/volume_server_handlers_read_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "strings" "testing" + "time" ) func TestInternalErrorWritesTheReadError(t *testing.T) { @@ -59,3 +60,39 @@ func TestShouldAttemptStreamWrite(t *testing.T) { } } } + +func TestNotModifiedChecksIfNoneMatchFirst(t *testing.T) { + const etag = "353dd8be" + const lastModified = 1700000000 + before := time.Unix(lastModified-60, 0).UTC().Format(http.TimeFormat) + after := time.Unix(lastModified+60, 0).UTC().Format(http.TimeFormat) + + tests := []struct { + name string + headers [][2]string + want bool + }{ + {"matching etag", [][2]string{{"If-None-Match", `"353dd8be"`}}, true}, + {"weak etag", [][2]string{{"If-None-Match", `W/"353dd8be"`}}, true}, + {"etag in a list", [][2]string{{"If-None-Match", `"x", "353dd8be"`}}, true}, + {"star", [][2]string{{"If-None-Match", `*`}}, true}, + {"other etag", [][2]string{{"If-None-Match", `"x"`}}, false}, + {"etag across repeated lines", [][2]string{{"If-None-Match", `"x"`}, {"If-None-Match", `"353dd8be"`}}, true}, + {"comma inside quotes", [][2]string{{"If-None-Match", `"353dd8be,old"`}}, false}, + {"other etag ignores If-Modified-Since", [][2]string{{"If-None-Match", `"x"`}, {"If-Modified-Since", after}}, false}, + {"empty If-None-Match ignores If-Modified-Since", [][2]string{{"If-None-Match", ``}, {"If-Modified-Since", after}}, false}, + {"not modified since", [][2]string{{"If-Modified-Since", after}}, true}, + {"modified since", [][2]string{{"If-Modified-Since", before}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest(http.MethodGet, "/3,01637037d6", nil) + for _, kv := range tt.headers { + r.Header.Add(kv[0], kv[1]) + } + if got := notModified(r, etag, lastModified); got != tt.want { + t.Errorf("notModified = %v, want %v", got, tt.want) + } + }) + } +} From 514c0c1e99d142e0e1ccdfa3323fed1963846915 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:05:46 -0700 Subject: [PATCH 62/70] topology: stop a stale duplicate-vid report from re-quarantining the survivor (#12032) * topology: drop a duplicate-vid report whose own copy is already gone flagDuplicateVolumeId runs on a goroutine after registration, so by the time it executes the reporter's copy may have been unregistered (a finished move drops the losing copy). The surviving copies are then legitimate, yet the stale report still quarantined them and stored a phantom (node, vid) key for the departed reporter. Skip the marks when the reporter no longer serves the volume under its own layout key. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(topology): cover a stale duplicate report against a lone survivor The existing tests only flag the conflict before the losing copy is unregistered. Run the delayed duplicate check after the reporter's copy is already gone and assert nothing is quarantined and the survivor stays writable, so removing the stale-report guard fails the test outright instead of needing a lucky schedule. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(topology): cover the enforceQuarantine-driven release retry A failed writable push re-arms the record, and the next volume report drains that retry through enforceQuarantine itself - a different entry point than the pulse-driven pass the other tests exercise. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/topology/duplicate_volume_id_test.go | 76 +++++++++++++++++++++++ weed/topology/topology.go | 10 ++- 2 files changed, 85 insertions(+), 1 deletion(-) diff --git a/weed/topology/duplicate_volume_id_test.go b/weed/topology/duplicate_volume_id_test.go index ec0ed4aeda..5fb315ea88 100644 --- a/weed/topology/duplicate_volume_id_test.go +++ b/weed/topology/duplicate_volume_id_test.go @@ -208,6 +208,29 @@ func TestQuarantineKeptWhenReplicaUnreadable(t *testing.T) { } } +func TestStaleDuplicateReportDoesNotQuarantineSurvivor(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + topo.RegisterVolumeLayout(v1, dn1) + + // dn2's copy is gone before its delayed duplicate check runs; the report + // still names dn2 but its layout no longer holds it. + topo.flagDuplicateVolumeId(v2, dn2) + + if topo.isQuarantined(dn1, v1.Id) || topo.isQuarantined(dn2, v2.Id) { + t.Fatal("a report for a copy that is already gone must not quarantine anything") + } + if !isWritable(t, topo, v1) { + t.Fatal("the surviving copy must stay writable") + } +} + func TestQuarantineSurvivesResync(t *testing.T) { topo := NewTopology("topo", nil, 128, 3, false) rp, _ := super_block.NewReplicaPlacementFromString("000") @@ -325,3 +348,56 @@ func TestDuplicateVolumeIdAcrossTtls(t *testing.T) { t.Fatal("same collection with a different ttl is still a reused volume id") } } + +func TestFailedWritablePushRetriedOnNextReport(t *testing.T) { + topo := NewTopology("topo", nil, 128, 3, false) + rp, _ := super_block.NewReplicaPlacementFromString("000") + + var failWritable atomic.Bool + var writablePushes atomic.Int32 + topo.QuarantineVolumeFn = func(dn *DataNode, vid needle.VolumeId, readOnly bool) error { + if !readOnly { + writablePushes.Add(1) + if failWritable.Load() { + return errors.New("volume server unreachable") + } + } + return nil + } + + dn1 := NewDataNode("server1") + dn2 := NewDataNode("server2") + v1 := storage.VolumeInfo{Id: 7, Collection: "a", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + v2 := storage.VolumeInfo{Id: 7, Collection: "b", ReplicaPlacement: rp, Ttl: needle.EMPTY_TTL} + dn1.UpdateVolumes([]storage.VolumeInfo{v1}) + dn2.UpdateVolumes([]storage.VolumeInfo{v2}) + // Register on the layouts directly: RegisterVolumeLayout would spawn an + // async flagDuplicateVolumeId per call, and a delayed one can re-quarantine + // the survivor at any later point in this test. + topo.GetVolumeLayout(v1.Collection, v1.ReplicaPlacement, v1.Ttl, types.ToDiskType(v1.DiskType)).RegisterVolume(&v1, dn1, nil) + topo.GetVolumeLayout(v2.Collection, v2.ReplicaPlacement, v2.Ttl, types.ToDiskType(v2.DiskType)).RegisterVolume(&v2, dn2, nil) + topo.flagDuplicateVolumeId(v2, dn2) + + // the conflict resolves while the survivor's writable push cannot land + failWritable.Store(true) + topo.UnRegisterVolumeLayout(v2, dn2) + require.Eventually(t, func() bool { + // The record reappears only after the failed writable push, and the + // restore arms it for retry in the same store - so this pair means the + // next report will take the retry branch rather than observing the + // release mid-flight. + return writablePushes.Load() >= 1 && topo.isQuarantined(dn1, v1.Id) + }, 5*time.Second, time.Millisecond, "a failed writable push must re-arm the record") + + reported := v1 + reported.ReadOnly = true + failWritable.Store(false) + topo.enforceQuarantine(dn1, reported) + + require.Eventually(t, func() bool { + // The release clears the record and the layout mark before the remote + // push lands, so the count has to be part of the condition or the poll + // can pass before the retried push runs. + return !topo.isQuarantined(dn1, v1.Id) && isWritable(t, topo, v1) && writablePushes.Load() >= 2 + }, 5*time.Second, time.Millisecond, "the next volume report must retry the release and its writable push") +} diff --git a/weed/topology/topology.go b/weed/topology/topology.go index 07f1ffa765..3ceda5de78 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -872,6 +872,14 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { if len(hits) == 0 { return } + // This runs on a goroutine after registration, so the reporter's copy may + // already be unregistered (a finished move drops the losing copy). The + // surviving copies are then legitimate, and flagging them quarantines a + // healthy volume. + own := t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType)) + if own == nil || !own.HasDataNode(v.Id, dn) { + return + } stats.MasterDuplicateVolumeIds.WithLabelValues(v.Collection, v.Id.String()).Inc() var otherKeys []string for _, h := range hits { @@ -913,7 +921,7 @@ func (t *Topology) flagDuplicateVolumeId(v storage.VolumeInfo, dn *DataNode) { t.releaseQuarantineWritable(n, v.Id) } } - quarantine(dn, t.FindVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl, types.ToDiskType(v.DiskType))) + quarantine(dn, own) for _, h := range hits { for _, n := range h.nodes { quarantine(n, h.vl) From 52e05ccda417c0a5c794e7e54a7cc83ca48c81e6 Mon Sep 17 00:00:00 2001 From: Wleeaf Date: Sat, 24 Oct 2026 17:06:20 +0300 Subject: [PATCH 63/70] s3: accept a self-copy that changes the storage class (#12031) * s3: accept a self-copy that changes the storage class or encryption AWS changes an object's storage class or encryption by copying the object onto itself with the new x-amz-storage-class or SSE headers, and only rejects a self-copy that changes none of metadata, storage class, website redirect location or encryption. CopyObject rejected every self-copy without REPLACE metadata or tagging directives as InvalidRequest, so these requests failed. * s3: reject an invalid x-amz-storage-class on CopyObject A self-copy with a bad storage class used to fail as InvalidRequest; once such copies are admitted the value must be validated like PutObject does, or it would be stored as opaque metadata. * test: cover the invalid storage class rejection on self-copy * s3: only a storage class change admits a self-copy Admitting encryption changes on a self-copy is unsafe: the metadata-only path compares only KMS key IDs, so a new encryption context or bucket-key setting is advertised over the old data key, and SSE-KMS key rotation onto the same entry keeps the old data key. Count only a requested storage class that differs from the object's (STANDARD when none is stored) as a change, so a self-copy to the current class or with only encryption headers is rejected as before. Such a self-copy updates the object in place and has to keep its encryption; one that would re-encrypt it or uses SSE-KMS is rejected too. --------- Co-authored-by: Wleeaf <179125857+wleeaf@users.noreply.github.com> Co-authored-by: user.email --- weed/s3api/s3api_object_handlers_copy.go | 27 ++++- ...object_handlers_copy_storage_class_test.go | 111 ++++++++++++++++++ 2 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 weed/s3api/s3api_object_handlers_copy_storage_class_test.go diff --git a/weed/s3api/s3api_object_handlers_copy.go b/weed/s3api/s3api_object_handlers_copy.go index 2473ed77cc..555dcaf696 100644 --- a/weed/s3api/s3api_object_handlers_copy.go +++ b/weed/s3api/s3api_object_handlers_copy.go @@ -70,6 +70,14 @@ func isValidDirective(value string) bool { return value == "" || value == DirectiveCopy || value == DirectiveReplace } +// copyChangesStorageClass reports whether a copy request asks for a storage +// class other than the source object's. That makes copying an object onto +// itself a change AWS accepts, even with the COPY directive. +func (s3a *S3ApiServer) copyChangesStorageClass(h http.Header, src *filer_pb.Entry) bool { + sc := h.Get(s3_constants.AmzStorageClass) + return sc != "" && sc != s3a.getStorageClassFromExtended(src.Extended) +} + // hasPrefixFold reports whether s starts with prefix, ignoring case. func hasPrefixFold(s, prefix string) bool { return len(s) >= len(prefix) && strings.EqualFold(s[:len(prefix)], prefix) @@ -196,6 +204,11 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request return } + if sc := r.Header.Get(s3_constants.AmzStorageClass); sc != "" && !validateStorageClass(sc) { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidStorageClass) + return + } + replaceMeta, replaceTagging := replaceDirective(r.Header) // If source object is empty or bucket is empty, reply back invalid copy source. @@ -266,9 +279,10 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request } sameDestination := srcBucket == dstBucket && srcObject == dstObject + changesStorageClass := s3a.copyChangesStorageClass(r.Header, entry) // A self-copy into a versioned bucket writes a new version instead of overwriting in // place, so it is not the no-op AWS rejects. It is how an earlier version is restored. - if sameDestination && !(replaceMeta || replaceTagging) && srcVersioningState != s3_constants.VersioningEnabled { + if sameDestination && !(replaceMeta || replaceTagging || changesStorageClass) && srcVersioningState != s3_constants.VersioningEnabled { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest) return } @@ -300,7 +314,16 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request replacesSource := copyReplacesSourceEntry(sameDestination, dstVersioningState, srcVersionId) - if replacesSource && (replaceMeta || replaceTagging) && s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject) { + // A self-copy that only changes the storage class updates the object in + // place and has to keep its encryption as stored. The direct strategy + // compares only KMS key IDs, so SSE-KMS requests stay rejected too. + if replacesSource && changesStorageClass && !(replaceMeta || replaceTagging) && + (IsSSEKMSRequest(r) || !s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject)) { + s3err.WriteErrorResponse(w, r, s3err.ErrInvalidCopyDest) + return + } + + if replacesSource && (replaceMeta || replaceTagging || changesStorageClass) && s3a.canUseMetadataOnlySelfCopy(entry, r, dstBucket, dstObject) { var dstVersionId string var etag string // Bind the entire self-copy update to the snapshot read on each attempt. diff --git a/weed/s3api/s3api_object_handlers_copy_storage_class_test.go b/weed/s3api/s3api_object_handlers_copy_storage_class_test.go new file mode 100644 index 0000000000..e176fe5daa --- /dev/null +++ b/weed/s3api/s3api_object_handlers_copy_storage_class_test.go @@ -0,0 +1,111 @@ +package s3api + +import ( + "encoding/base64" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/s3_pb" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "google.golang.org/protobuf/proto" +) + +// AWS changes an object's storage class by copying the object onto itself +// with the COPY directive and a new x-amz-storage-class. Only a self-copy that +// changes nothing is rejected, and a bucket's default encryption is no change. +// A self-copy that would also change the encryption stays rejected. +func TestCopyObjectToItselfChangingStorageClass(t *testing.T) { + const bucket, object = "self-copy", "object" + kmsObject := map[string]string{ + s3_constants.AmzServerSideEncryption: s3_constants.SSEAlgorithmKMS, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "key-a", + s3_constants.AmzServerSideEncryptionContext: base64.StdEncoding.EncodeToString([]byte(`{"env":"old"}`)), + } + for _, tt := range []struct { + name string + stored map[string]string // extra object metadata + storageClass string + headers map[string]string // extra request headers + defaultEncryption bool + wantCode int + wantErr string + }{ + {name: "no change", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "no change under default encryption", defaultEncryption: true, wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "invalid storage class", storageClass: "NOT_A_CLASS", wantCode: http.StatusBadRequest, wantErr: "InvalidStorageClass"}, + {name: "new storage class", storageClass: "STANDARD_IA", wantCode: http.StatusOK}, + {name: "current storage class", stored: map[string]string{s3_constants.AmzStorageClass: "STANDARD_IA"}, + storageClass: "STANDARD_IA", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "STANDARD without a stored class", storageClass: "STANDARD", wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "encryption only", headers: map[string]string{s3_constants.AmzServerSideEncryption: "AES256"}, + wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + {name: "new storage class and SSE-KMS context", stored: kmsObject, storageClass: "STANDARD_IA", headers: map[string]string{ + s3_constants.AmzServerSideEncryption: s3_constants.SSEAlgorithmKMS, + s3_constants.AmzServerSideEncryptionAwsKmsKeyId: "key-a", + s3_constants.AmzServerSideEncryptionContext: base64.StdEncoding.EncodeToString([]byte(`{"env":"new"}`)), + }, wantCode: http.StatusBadRequest, wantErr: "InvalidRequest"}, + } { + t.Run(tt.name, func(t *testing.T) { + filer := ©ACLFiler{ambiguousPutFiler: &ambiguousPutFiler{apply: true, entries: map[string]*filer_pb.Entry{}}} + s3a := newPutTestServer(t, startFakeFiler(t, filer)) + s3a.iam = &IdentityAccessManagement{} + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + config := &BucketConfig{Name: bucket, Ownership: s3_constants.OwnershipObjectWriter} + if tt.defaultEncryption { + config.Encryption = &s3_pb.EncryptionConfiguration{SseAlgorithm: s3_constants.SSEAlgorithmAES256} + } + s3a.bucketConfigCache.Set(bucket, config) + objectPath := "/buckets/" + bucket + "/" + object + original := &filer_pb.Entry{Name: object, Content: []byte("data"), + Attributes: &filer_pb.FuseAttributes{FileMode: defaultFileMode, FileSize: 4, Mime: "text/plain"}, + Extended: map[string][]byte{"X-Amz-Meta-Owner": []byte("kept")}} + for k, v := range tt.stored { + original.Extended[k] = []byte(v) + } + filer.entries[objectPath] = proto.Clone(original).(*filer_pb.Entry) + + req := httptest.NewRequest(http.MethodPut, "/"+bucket+"/"+object, nil) + req = mux.SetURLVars(req, map[string]string{"bucket": bucket, "object": object}) + req.Header.Set("X-Amz-Copy-Source", "/"+bucket+"/"+object) + req.Header.Set(s3_constants.AmzUserMetaDirective, DirectiveCopy) + if tt.storageClass != "" { + req.Header.Set(s3_constants.AmzStorageClass, tt.storageClass) + } + for k, v := range tt.headers { + req.Header.Set(k, v) + } + rr := httptest.NewRecorder() + s3a.CopyObjectHandler(rr, req) + + if rr.Code != tt.wantCode { + t.Fatalf("status = %d, want %d; body: %s", rr.Code, tt.wantCode, rr.Body.String()) + } + filer.mu.Lock() + defer filer.mu.Unlock() + stored := filer.entries[objectPath] + if tt.wantCode != http.StatusOK { + if !strings.Contains(rr.Body.String(), ""+tt.wantErr+"") { + t.Fatalf("body = %s, want %s", rr.Body.String(), tt.wantErr) + } + if !proto.Equal(stored, original) { + t.Fatalf("rejected copy changed the object: %v", stored) + } + return + } + if got := string(stored.Extended[s3_constants.AmzStorageClass]); got != tt.storageClass { + t.Fatalf("storage class = %q, want %q", got, tt.storageClass) + } + if got := string(stored.Extended["X-Amz-Meta-Owner"]); got != "kept" { + t.Fatalf("user metadata = %q, want it kept under the COPY directive", got) + } + if got := string(stored.Content); got != "data" { + t.Fatalf("content = %q, want %q", got, "data") + } + }) + } +} From 18f9d9a0b690559772015ac8f23e11cf1da8bcc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A7=D0=B5=D0=BF=D1=83=D1=80=D0=BD=D0=BE=D0=B2=20=D0=A1?= =?UTF-8?q?=D0=B5=D0=BC=D1=91=D0=BD?= Date: Sat, 24 Oct 2026 20:06:46 +0600 Subject: [PATCH 64/70] feat: Free Volume Deadband (M) for EC shard placement (#12029) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Free Volume Deadband (M) for EC Place nodes and racks |ΔfreeSlots| ≤ M is a tie; remainder is liveShards. M=0 stays strict freeSlots. M is free volume slots (× dataShards). pickNodeInRackEligible and pickRackEligible share the rule. Worker plugin config free_slots_deadband, default 0. Not on the write path. * fix: anchor EC freeSlots deadband on class maxFree Pairwise |Δfree|≤M vs the current winner is not transitive: 120→110→100 at M=10 slots picked 100. After evenness, rank by maxFree, keep |free-maxFree|≤M, then min liveShards. Shared by node and rack. M=0 is still only maxFree then live. * worker: add free_slots_deadband to ErasureCodingTaskConfig * erasure_coding: persist free_slots_deadband through TaskPolicy ApplyMaintenanceConfigFromToml accepted the key but the admin save path round-trips config through ToTaskPolicy, so the value was dropped on reload; serialize it like the other EC settings. * erasure_coding: bound free_slots_deadband at its declared maximum The TOML loader and the plugin worker config only clamped negatives, so a value above the spec's 100000 could reach the int32 TaskPolicy field and wrap; cap at 100000 on both entry paths and declare MaxValue on the plugin field. * erasure_coding: document free_slots_deadband in scaffold admin.toml --------- Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com> Co-authored-by: user.email --- weed/admin/dash/config_toml.go | 22 +- weed/command/scaffold/admin.toml | 3 + weed/pb/worker.proto | 1 + weed/pb/worker_pb/worker.pb.go | 33 ++- .../erasure_coding/ecbalancer/balancer.go | 3 +- .../erasure_coding/ecbalancer/place.go | 178 ++++++++++--- .../erasure_coding/ecbalancer/place_test.go | 233 ++++++++++++++++++ weed/worker/tasks/erasure_coding/config.go | 47 ++-- weed/worker/tasks/erasure_coding/detection.go | 11 +- .../erasure_coding/detection_strict_test.go | 4 + .../tasks/erasure_coding/plugin_handler.go | 32 ++- .../erasure_coding/plugin_handler_test.go | 56 +++++ 12 files changed, 551 insertions(+), 72 deletions(-) diff --git a/weed/admin/dash/config_toml.go b/weed/admin/dash/config_toml.go index 92532963ef..bdc00a7328 100644 --- a/weed/admin/dash/config_toml.go +++ b/weed/admin/dash/config_toml.go @@ -102,6 +102,15 @@ func (cp *ConfigPersistence) ApplyMaintenanceConfigFromToml(v TomlConfig) error ecConf.StrictPlacement = v.GetBool(k) ecChanged = true } + if k := "maintenance.erasure_coding.free_slots_deadband"; v.IsSet(k) { + ecConf.FreeSlotsDeadband = v.GetInt(k) + if ecConf.FreeSlotsDeadband < 0 { + ecConf.FreeSlotsDeadband = 0 + } else if ecConf.FreeSlotsDeadband > 100000 { + ecConf.FreeSlotsDeadband = 100000 + } + ecChanged = true + } if !maintenanceChanged && !vacuumChanged && !balanceChanged && !ecChanged { return nil @@ -223,12 +232,13 @@ var pluginConfigSections = []pluginConfigSection{ jobType: "erasure_coding", prefix: "maintenance.erasure_coding", workerKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{ - "fullness_ratio": doubleValue, - "quiet_for_seconds": int64Value, - "min_size_mb": int64Value, - "preferred_tags": stringListValue, - "replica_placement": stringValue, - "strict_placement": boolValue, + "fullness_ratio": doubleValue, + "quiet_for_seconds": int64Value, + "min_size_mb": int64Value, + "preferred_tags": stringListValue, + "replica_placement": stringValue, + "strict_placement": boolValue, + "free_slots_deadband": int64Value, }, // workers read collection_filter from the admin values, not the worker values adminKeys: map[string]func(v TomlConfig, key string) *plugin_pb.ConfigValue{ diff --git a/weed/command/scaffold/admin.toml b/weed/command/scaffold/admin.toml index e1af67d7a7..99709abc03 100644 --- a/weed/command/scaffold/admin.toml +++ b/weed/command/scaffold/admin.toml @@ -66,6 +66,9 @@ # replica_placement = "" # fail planning when the placement constraints cannot be met instead of relaxing them # strict_placement = false +# volume-slot deadband for shard placement: nodes whose free volume slots differ by at +# most this much tie and fewer live shards wins; applied times data shards (0 = strict) +# free_slots_deadband = 0 # max retry attempts for a failed erasure coding job # retry_limit = 1 # seconds to wait between retry attempts diff --git a/weed/pb/worker.proto b/weed/pb/worker.proto index fa06886d09..822f89269c 100644 --- a/weed/pb/worker.proto +++ b/weed/pb/worker.proto @@ -381,6 +381,7 @@ message ErasureCodingTaskConfig { repeated string preferred_tags = 5; // Disk tags to prioritize for EC shard placement string replica_placement = 6; // EC shard replica placement (e.g. "020"); empty falls back to master default replication bool strict_placement = 7; // fail planning instead of relaxing placement constraints + int32 free_slots_deadband = 8; // Place free-volume-slot deadband; |ΔfreeSlots| ≤ M×dataShards ranks by liveShards; 0 = strict } // BalanceTaskConfig contains balance-specific configuration diff --git a/weed/pb/worker_pb/worker.pb.go b/weed/pb/worker_pb/worker.pb.go index a516babcce..e2a81e3f5f 100644 --- a/weed/pb/worker_pb/worker.pb.go +++ b/weed/pb/worker_pb/worker.pb.go @@ -2973,16 +2973,17 @@ func (x *VacuumTaskConfig) GetMinIntervalSeconds() int32 { // ErasureCodingTaskConfig contains EC-specific configuration type ErasureCodingTaskConfig struct { - state protoimpl.MessageState `protogen:"open.v1"` - FullnessRatio float64 `protobuf:"fixed64,1,opt,name=fullness_ratio,json=fullnessRatio,proto3" json:"fullness_ratio,omitempty"` // Minimum fullness ratio to trigger EC (0.0-1.0) - QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC - MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC - CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections - PreferredTags []string `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"` // Disk tags to prioritize for EC shard placement - ReplicaPlacement string `protobuf:"bytes,6,opt,name=replica_placement,json=replicaPlacement,proto3" json:"replica_placement,omitempty"` // EC shard replica placement (e.g. "020"); empty falls back to master default replication - StrictPlacement bool `protobuf:"varint,7,opt,name=strict_placement,json=strictPlacement,proto3" json:"strict_placement,omitempty"` // fail planning instead of relaxing placement constraints - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + FullnessRatio float64 `protobuf:"fixed64,1,opt,name=fullness_ratio,json=fullnessRatio,proto3" json:"fullness_ratio,omitempty"` // Minimum fullness ratio to trigger EC (0.0-1.0) + QuietForSeconds int32 `protobuf:"varint,2,opt,name=quiet_for_seconds,json=quietForSeconds,proto3" json:"quiet_for_seconds,omitempty"` // Minimum quiet time before EC + MinVolumeSizeMb int32 `protobuf:"varint,3,opt,name=min_volume_size_mb,json=minVolumeSizeMb,proto3" json:"min_volume_size_mb,omitempty"` // Minimum volume size for EC + CollectionFilter string `protobuf:"bytes,4,opt,name=collection_filter,json=collectionFilter,proto3" json:"collection_filter,omitempty"` // Only process volumes from specific collections + PreferredTags []string `protobuf:"bytes,5,rep,name=preferred_tags,json=preferredTags,proto3" json:"preferred_tags,omitempty"` // Disk tags to prioritize for EC shard placement + ReplicaPlacement string `protobuf:"bytes,6,opt,name=replica_placement,json=replicaPlacement,proto3" json:"replica_placement,omitempty"` // EC shard replica placement (e.g. "020"); empty falls back to master default replication + StrictPlacement bool `protobuf:"varint,7,opt,name=strict_placement,json=strictPlacement,proto3" json:"strict_placement,omitempty"` // fail planning instead of relaxing placement constraints + FreeSlotsDeadband int32 `protobuf:"varint,8,opt,name=free_slots_deadband,json=freeSlotsDeadband,proto3" json:"free_slots_deadband,omitempty"` // Place free-volume-slot deadband; |ΔfreeSlots| ≤ M×dataShards ranks by liveShards; 0 = strict + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ErasureCodingTaskConfig) Reset() { @@ -3064,6 +3065,13 @@ func (x *ErasureCodingTaskConfig) GetStrictPlacement() bool { return false } +func (x *ErasureCodingTaskConfig) GetFreeSlotsDeadband() int32 { + if x != nil { + return x.FreeSlotsDeadband + } + return 0 +} + // BalanceTaskConfig contains balance-specific configuration type BalanceTaskConfig struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -4293,7 +4301,7 @@ const file_worker_proto_rawDesc = "" + "\x10VacuumTaskConfig\x12+\n" + "\x11garbage_threshold\x18\x01 \x01(\x01R\x10garbageThreshold\x12/\n" + "\x14min_volume_age_hours\x18\x02 \x01(\x05R\x11minVolumeAgeHours\x120\n" + - "\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xc5\x02\n" + + "\x14min_interval_seconds\x18\x03 \x01(\x05R\x12minIntervalSeconds\"\xf5\x02\n" + "\x17ErasureCodingTaskConfig\x12%\n" + "\x0efullness_ratio\x18\x01 \x01(\x01R\rfullnessRatio\x12*\n" + "\x11quiet_for_seconds\x18\x02 \x01(\x05R\x0fquietForSeconds\x12+\n" + @@ -4301,7 +4309,8 @@ const file_worker_proto_rawDesc = "" + "\x11collection_filter\x18\x04 \x01(\tR\x10collectionFilter\x12%\n" + "\x0epreferred_tags\x18\x05 \x03(\tR\rpreferredTags\x12+\n" + "\x11replica_placement\x18\x06 \x01(\tR\x10replicaPlacement\x12)\n" + - "\x10strict_placement\x18\a \x01(\bR\x0fstrictPlacement\"\x9b\x01\n" + + "\x10strict_placement\x18\a \x01(\bR\x0fstrictPlacement\x12.\n" + + "\x13free_slots_deadband\x18\b \x01(\x05R\x11freeSlotsDeadband\"\x9b\x01\n" + "\x11BalanceTaskConfig\x12/\n" + "\x13imbalance_threshold\x18\x01 \x01(\x01R\x12imbalanceThreshold\x12(\n" + "\x10min_server_count\x18\x02 \x01(\x05R\x0eminServerCount\x12+\n" + diff --git a/weed/storage/erasure_coding/ecbalancer/balancer.go b/weed/storage/erasure_coding/ecbalancer/balancer.go index 0c8541fea6..2e8ad0ffa5 100644 --- a/weed/storage/erasure_coding/ecbalancer/balancer.go +++ b/weed/storage/erasure_coding/ecbalancer/balancer.go @@ -1284,7 +1284,8 @@ func volumeShardCount(node *Node, vk volKey) int { } // nodeLiveShardCount is the disk-level EC shard cardinality (master EcShards), -// not the UI "EC volumes" count (len of the volume map). +// not the UI "EC volumes" count (len of the volume map). Place uses it as the +// remainder after even-this-volume and freeSlots (with deadband). func nodeLiveShardCount(n *Node) int { if n == nil { return 0 diff --git a/weed/storage/erasure_coding/ecbalancer/place.go b/weed/storage/erasure_coding/ecbalancer/place.go index c9d8e438ef..6ea7e84a72 100644 --- a/weed/storage/erasure_coding/ecbalancer/place.go +++ b/weed/storage/erasure_coding/ecbalancer/place.go @@ -2,6 +2,7 @@ package ecbalancer import ( "fmt" + "math" "sort" "strings" @@ -28,6 +29,11 @@ type Constraints struct { PreferredTags []string ReplicaPlacement *super_block.ReplicaPlacement Ratio func(collection string) (dataShards, parityShards int) + // FreeSlotsDeadband (M) is free volume slots. After even-this-volume, + // Place ranks that evenness class by maxFree, keeps |free−maxFree| ≤ + // M×dataShards, then lower liveShards. 0 is only maxFree then live + // (old strict freeSlots; liveShards only on an exact free tie). + FreeSlotsDeadband int } // DiskTypePolicy controls how Constraints.DiskType constrains placement. @@ -193,7 +199,7 @@ func (t *Topology) Place(vid uint32, collection string, need []int, c Constraint eligible := func(d *disk) bool { return typeEligible(d) && (len(tt) == 0 || diskHasAnyTag(d, tt)) } - res, err := t.tryPlace(vk, need, dataShards, parityShards, racks, rackKeys, mode, c.ReplicaPlacement, eligible, prefer) + res, err := t.tryPlace(vk, need, dataShards, parityShards, racks, rackKeys, mode, c.ReplicaPlacement, eligible, prefer, volumeDeadbandToShardSlots(c.FreeSlotsDeadband, dataShards)) if err != nil { lastErr = err continue @@ -223,7 +229,7 @@ func tagTiers(preferredTags []string) [][]string { // `eligible`, with `prefer` (may be nil) ranking soft-preferred disks first. It // journals reservations and rolls them all back if any shard cannot be placed, so // a failed tier leaves the snapshot unchanged for the next attempt. -func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, racks map[string]*rack, rackKeys []string, mode PlacementMode, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool) (*PlaceResult, error) { +func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, racks map[string]*rack, rackKeys []string, mode PlacementMode, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, freeSlotsDeadband int) (*PlaceResult, error) { result := &PlaceResult{Destinations: make(map[int]Destination, len(need))} // Per-type shard ids per rack (even caps), total shard count per rack @@ -297,7 +303,7 @@ func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, typeTotal = parityShards } for _, rl := range attempts { - node, diskID, spilled, ok := chooseShardDest(vk, sid, isData, dataShards, typeTotal, numEligibleRacks, parityShards, rackCap(rl), racks, rackKeys, rp, eligible, prefer, shardsPerRack[isData], rackShardCount, bearing, rl) + node, diskID, spilled, ok := chooseShardDest(vk, sid, isData, dataShards, typeTotal, numEligibleRacks, parityShards, rackCap(rl), racks, rackKeys, rp, eligible, prefer, shardsPerRack[isData], rackShardCount, bearing, rl, freeSlotsDeadband) if !ok { continue } @@ -350,12 +356,13 @@ func (t *Topology) tryPlace(vk volKey, need []int, dataShards, parityShards int, // chooseShardDest selects a (node, disk) for one shard at the given relaxation // level: pick a rack (total-shards-per-rack cap + even per-type cap + -// ReplicaPlacement caps + two-pass anti-affinity to the opposite type), then the -// least-loaded eligible node, then the best eligible disk. If no node in the -// chosen rack fits (e.g. every node is at SameRackCount), the next-best rack is -// tried. The third return reports whether the disk spilled off the -// soft-preferred type. ok=false when no rack/node/disk fits. -func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, numEligibleRacks, maxPerDisk, maxTotalPerRack int, racks map[string]*rack, rackKeys []string, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, shardsPerRackType map[string][]int, rackShardCount map[string]int, bearing map[bool]map[string]bool, rl relaxation) (*Node, uint32, bool, bool) { +// ReplicaPlacement caps + two-pass anti-affinity to the opposite type, then +// freeSlots deadband / liveShards), then the least-loaded eligible node, then +// the best eligible disk. If no node in the chosen rack fits (e.g. every node +// is at SameRackCount), the next-best rack is tried. The third return reports +// whether the disk spilled off the soft-preferred type. ok=false when no +// rack/node/disk fits. +func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, numEligibleRacks, maxPerDisk, maxTotalPerRack int, racks map[string]*rack, rackKeys []string, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, prefer func(*disk) bool, shardsPerRackType map[string][]int, rackShardCount map[string]int, bearing map[bool]map[string]bool, rl relaxation, freeSlotsDeadband int) (*Node, uint32, bool, bool) { maxPerRack := numEligibleRacks*typeTotal + 1 // effectively unlimited when caps are relaxed if rl.caps { if maxPerRack = ceilDivide(typeTotal, numEligibleRacks); maxPerRack < 1 { @@ -389,12 +396,11 @@ func chooseShardDest(vk volKey, sid int, isData bool, dataShards, typeTotal, num return !tried[r] && racks[r].freeSlots > 0 && rackShardRoom(racks[r], vk, eligible, maxPerDisk) > 0 } for { - destRack, ok := pickTarget(rackKeys, shardsPerRackType, maxPerRack, anti, hasRoom, withinLimit, - func(r string) int { return racks[r].freeSlots }) + destRack, ok := pickRackEligible(rackKeys, shardsPerRackType, maxPerRack, anti, hasRoom, withinLimit, racks, freeSlotsDeadband) if !ok { return nil, 0, false, false } - if node := pickNodeInRackEligible(racks[destRack], vk, rp, eligible, maxPerDisk); node != nil { + if node := pickNodeInRackEligible(racks[destRack], vk, rp, eligible, maxPerDisk, freeSlotsDeadband); node != nil { if diskID, ok, spilled := pickBestDiskEligible(node, vk, eligible, prefer, sid, dataShards, maxPerDisk); ok { return node, diskID, spilled, true } @@ -482,18 +488,83 @@ func rackShardRoomUnderRP(r *rack, vk volKey, eligible func(*disk) bool, maxPerD return room } +// pickRackEligible is pickTarget for Place racks. Even this-type / cap / +// two-pass anti-affinity stay. Among that evenness class the remainder is +// max rack freeSlots, keep |free−maxFree| ≤ shard-slot deadband, then +// lower liveShards (sum of nodeLiveShardCount). M=0 is only maxFree then +// live (old strict freeSlots). pickTarget itself is unchanged so balancer +// / DC paths stay without M. +func pickRackEligible(candidates []string, shardsPerTarget map[string][]int, maxPerTarget int, antiAffinity map[string]bool, hasFreeSlots, withinLimit func(string) bool, racks map[string]*rack, freeSlotsDeadband int) (string, bool) { + try := func(skipAnti bool) (string, bool) { + var keys []string + var free, live []int + bestCount := maxPerTarget + 1 + for _, c := range candidates { + if skipAnti && antiAffinity[c] { + continue + } + if !hasFreeSlots(c) { + continue + } + if len(shardsPerTarget[c]) >= maxPerTarget { + continue + } + if !withinLimit(c) { + continue + } + cnt := len(shardsPerTarget[c]) + if len(keys) == 0 || cnt < bestCount { + keys, free, live = keys[:0], free[:0], live[:0] + bestCount = cnt + } else if cnt > bestCount { + continue + } + f := 0 + if r := racks[c]; r != nil { + f = r.freeSlots + } + keys = append(keys, c) + free = append(free, f) + live = append(live, rackLiveShardCount(racks[c])) + } + if len(keys) == 0 { + return "", false + } + return keys[pickDeadbandRemainder(free, live, freeSlotsDeadband)], true + } + if len(antiAffinity) > 0 { + if t, ok := try(true); ok { + return t, true + } + } + return try(false) +} + +func rackLiveShardCount(r *rack) int { + if r == nil { + return 0 + } + total := 0 + for _, n := range r.nodes { + total += nodeLiveShardCount(n) + } + return total +} + // pickNodeInRackEligible is pickNodeInRack restricted to nodes with an eligible // disk that can take another shard of the volume (free slot, under maxPerDisk). // FromActiveTopology keeps all disk types/tags in the snapshot, so without this a // node with free volume slots but no eligible disk could be chosen. // -// Even first (this volume, machine then node). Machine freeSlots remainder -// (mFree) was dropped: keep host even (mCount); remainder is disk freeSlots -// then liveShards so thin disks fill first. SameRackCount / 2+2 unchanged. -func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, maxPerDisk int) *Node { +// Even first (this volume, machine then node). Among that class: max node +// freeSlots, keep |free−maxFree| ≤ shard-slot deadband, then lower +// liveShards. Machine freeSlots (mFree) stays dropped as on origin/master. +// SameRackCount / 2+2 unchanged. M=0 is only maxFree then live. Callers +// convert UI volume M via volumeDeadbandToShardSlots. +func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement, eligible func(*disk) bool, maxPerDisk, freeSlotsDeadband int) *Node { machineShards := countShardsByHost(vk, r.nodes) - var best *Node - var bestMCount, bestNCount, bestNFree, bestNScore int + var nodes []*Node + var bestMCount, bestNCount int for _, id := range sortedNodeKeys(r.nodes) { node := r.nodes[id] if nodeShardRoom(node, vk, eligible, maxPerDisk) <= 0 { @@ -504,27 +575,68 @@ func pickNodeInRackEligible(r *rack, vk volKey, rp *super_block.ReplicaPlacement continue } mCount := machineShards[node.host] - nScore := nodeLiveShardCount(node) - better := false - switch { - case best == nil: - better = true - case mCount != bestMCount: - better = mCount < bestMCount - case count != bestNCount: - better = count < bestNCount - case node.freeSlots != bestNFree: - better = node.freeSlots > bestNFree - default: - better = nScore < bestNScore + if len(nodes) == 0 || mCount < bestMCount || (mCount == bestMCount && count < bestNCount) { + nodes = nodes[:0] + bestMCount, bestNCount = mCount, count + } else if mCount != bestMCount || count != bestNCount { + continue } - if better { - best, bestMCount, bestNCount, bestNFree, bestNScore = node, mCount, count, node.freeSlots, nScore + nodes = append(nodes, node) + } + if len(nodes) == 0 { + return nil + } + free := make([]int, len(nodes)) + live := make([]int, len(nodes)) + for i, n := range nodes { + free[i] = n.freeSlots + live[i] = nodeLiveShardCount(n) + } + return nodes[pickDeadbandRemainder(free, live, freeSlotsDeadband)] +} + +// pickDeadbandRemainder ranks same-evenness candidates: maxFree of the +// class, keep |free−maxFree|≤m, then min live. Ties keep the earlier +// index (scan order). M=0 keeps only maxFree then live. Shared by node +// and rack so the band cannot drift. +func pickDeadbandRemainder(free, live []int, m int) int { + maxFree := free[0] + for _, f := range free[1:] { + if f > maxFree { + maxFree = f + } + } + best := -1 + for i, f := range free { + if absIntDiff(f, maxFree) > m { + continue + } + if best < 0 || live[i] < live[best] { + best = i } } return best } +func absIntDiff(a, b int) int { + if a > b { + return a - b + } + return b - a +} + +// volumeDeadbandToShardSlots converts UI volume-slot M to the shard-slot +// deadband Place compares against node.freeSlots. OSS 10+4 uses dataShards=10. +func volumeDeadbandToShardSlots(volumes, dataShards int) int { + if volumes <= 0 || dataShards <= 0 { + return 0 + } + if volumes > math.MaxInt/dataShards { + return math.MaxInt + } + return volumes * dataShards +} + // pickBestDiskEligible chooses the best eligible disk on a node, ranking // soft-preferred disks (prefer != nil && prefer(d)) ahead of others so disk-type // Prefer uses the preferred type when available but spills otherwise. Returns the diff --git a/weed/storage/erasure_coding/ecbalancer/place_test.go b/weed/storage/erasure_coding/ecbalancer/place_test.go index cf8f47f2ee..32eef6edba 100644 --- a/weed/storage/erasure_coding/ecbalancer/place_test.go +++ b/weed/storage/erasure_coding/ecbalancer/place_test.go @@ -2,6 +2,7 @@ package ecbalancer import ( "fmt" + "math" "sort" "testing" @@ -914,3 +915,235 @@ func TestPlace10Plus4ImbalanceTiebreak(t *testing.T) { }) } } + +func TestVolumeDeadbandToShardSlots(t *testing.T) { + if got := volumeDeadbandToShardSlots(0, 10); got != 0 { + t.Fatalf("M=0 → %d, want 0", got) + } + if got := volumeDeadbandToShardSlots(1, 10); got != 10 { + t.Fatalf("M=1 at 10+4 → %d, want 10", got) + } + if got := volumeDeadbandToShardSlots(1, 8); got != 8 { + t.Fatalf("M=1 at 8+4 → %d, want 8", got) + } + if got := volumeDeadbandToShardSlots(50, 10); got != 500 { + t.Fatalf("M=50 at 10+4 → %d, want 500", got) + } + if got := volumeDeadbandToShardSlots(-3, 10); got != 0 { + t.Fatalf("negative M → %d, want 0", got) + } +} + +func TestAbsIntDiff(t *testing.T) { + if got := absIntDiff(105, 100); got != 5 { + t.Fatalf("absIntDiff(105, 100)=%d, want 5", got) + } + if got := absIntDiff(100, 105); got != 5 { + t.Fatalf("absIntDiff(100, 105)=%d, want 5", got) + } + if got := absIntDiff(100, 100); got != 0 { + t.Fatalf("absIntDiff(100, 100)=%d, want 0", got) + } + if got := absIntDiff(math.MaxInt, math.MaxInt-5); got != 5 { + t.Fatalf("absIntDiff(MaxInt, MaxInt-5)=%d, want 5", got) + } + if got := absIntDiff(math.MaxInt-5, math.MaxInt); got != 5 { + t.Fatalf("absIntDiff(MaxInt-5, MaxInt)=%d, want 5", got) + } + if got := absIntDiff(math.MinInt+5, math.MinInt); got != 5 { + t.Fatalf("absIntDiff(MinInt+5, MinInt)=%d, want 5", got) + } + if got := absIntDiff(math.MinInt, math.MinInt+5); got != 5 { + t.Fatalf("absIntDiff(MinInt, MinInt+5)=%d, want 5", got) + } +} + +func placeFirstShardNode(t *testing.T, topo *Topology, m int) string { + t.Helper() + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: m}, PlaceStrict) + if err != nil { + t.Fatalf("Place M=%d: %v", m, err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + return d.Node +} + +// TestFreeSlotsDeadbandDefersToLiveShards: two nodes, empty this-volume. +// b has slightly more freeSlots but more live shards. M=0 follows freeSlots +// (b); M=1 volume (×10 shard slots at 10+4) covers the gap and follows +// liveShards (a). Sorted-id would always pick a. +func TestFreeSlotsDeadbandDefersToLiveShards(t *testing.T) { + build := func() *Topology { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 105) + b.AddDisk(0, "", 105, 50) + return topo + } + if got := placeFirstShardNode(t, build(), 0); got != "b:8081" { + t.Fatalf("M=0: got %s, want b (strict freeSlots)", got) + } + if got := placeFirstShardNode(t, build(), 1); got != "a:8080" { + t.Fatalf("M=1 volume: got %s, want a (liveShards remainder)", got) + } +} + +// TestFreeSlotsDeadbandZeroExactTieUsesLiveShards: equal freeSlots, M=0 +// remainder is liveShards, not the lower sorted id. +func TestFreeSlotsDeadbandZeroExactTieUsesLiveShards(t *testing.T) { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 50) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 100) + b.AddDisk(0, "", 100, 10) + if got := placeFirstShardNode(t, topo, 0); got != "b:8081" { + t.Fatalf("equal freeSlots M=0: got %s, want b (lower liveShards)", got) + } +} + +// TestFreeSlotsDeadbandStillHonorsLargeGap: gap above M still follows freeSlots +// even if that node is fatter on liveShards. +func TestFreeSlotsDeadbandStillHonorsLargeGap(t *testing.T) { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 200) + b.AddDisk(0, "", 200, 80) + if got := placeFirstShardNode(t, topo, 1); got != "b:8081" { + t.Fatalf("M=1 volume gap=100 shards: got %s, want b (freeSlots still wins)", got) + } +} + +// TestFreeSlotsDeadbandScalesWithRatio: M=1 volume is 8 shard slots at 8+4 +// and 10 at default 10+4. Gap of 9 shards: 8+4 still follows freeSlots; 10+4 +// ties and follows liveShards. +func TestFreeSlotsDeadbandScalesWithRatio(t *testing.T) { + build := func() *Topology { + topo := NewTopology() + a := topo.AddNode("a:8080", "dc1", "dc1:rack0", 100) + a.AddDisk(0, "", 100, 10) + b := topo.AddNode("b:8081", "dc1", "dc1:rack0", 109) + b.AddDisk(0, "", 109, 50) + return topo + } + place := func(topo *Topology, ratio func(string) (int, int)) string { + t.Helper() + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: 1, Ratio: ratio}, PlaceStrict) + if err != nil { + t.Fatalf("Place: %v", err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + return d.Node + } + if got := place(build(), func(string) (int, int) { return 8, 4 }); got != "b:8081" { + t.Fatalf("8+4 M=1 gap=9: got %s, want b (deadband 8, freeSlots)", got) + } + if got := place(build(), nil); got != "a:8080" { + t.Fatalf("10+4 M=1 gap=9: got %s, want a (deadband 10, liveShards)", got) + } +} + +func rackExtraTopo(thinFree, thinLive, fatFree, fatLive int) *Topology { + topo := NewTopology() + add := func(rack, id string, free, live int) { + n := topo.AddNode(id, "dc1", rack, free) + n.AddDisk(0, "", free, live) + } + add("dc1:rack0", "thin:8080", thinFree, thinLive) + add("dc1:rack1", "fat1:8080", fatFree, fatLive) + add("dc1:rack2", "fat2:8080", fatFree, fatLive) + add("dc1:rack3", "fat3:8080", fatFree, fatLive) + return topo +} + +func place104PerRack(t *testing.T, topo *Topology, m int) map[string]int { + t.Helper() + res, err := topo.Place(1, "c1", allShards(), Constraints{FreeSlotsDeadband: m}, PlaceStrict) + if err != nil { + t.Fatalf("Place M=%d: %v", m, err) + } + got := map[string]int{} + for _, d := range res.Destinations { + got[d.Rack]++ + } + return got +} + +// TestRackDeadbandDefersToLiveShards: 10+4 extras are 4+4+3+3. Thin rack has +// fewer freeSlots (gap 5) but fewer liveShards. M=1 volume is 10 shard slots, +// so the gap is a tie and extras go to the thinner live rack. +func TestRackDeadbandDefersToLiveShards(t *testing.T) { + got := place104PerRack(t, rackExtraTopo(100, 10, 105, 50), 1) + if got["rack0"] != 4 { + t.Fatalf("M=1 gap=5: thin rack0 got %d, want 4 (liveShards remainder); per-rack %v", got["rack0"], got) + } +} + +// TestRackDeadbandStillHonorsLargeGap: same thin-vs-fat live, but freeSlots +// gap 100 is above M=1 (10 shard slots), so extras still follow freeSlots. +func TestRackDeadbandStillHonorsLargeGap(t *testing.T) { + got := place104PerRack(t, rackExtraTopo(100, 10, 200, 50), 1) + if got["rack0"] != 3 { + t.Fatalf("M=1 gap=100: thin rack0 got %d, want 3 (freeSlots still wins); per-rack %v", got["rack0"], got) + } +} + +// Greptile chain at M=1 volume (10 shard slots): 120/300, 110/200, 100/100. +// Pairwise vs the current winner walks 120→110→100 or 100→110→120 depending +// on scan order. maxFree=120 keeps |Δ|≤10, so 110 wins — not 100 (out of +// band) and not 120 (fatter live). +func TestNodeDeadbandAnchorsOnMaxFree(t *testing.T) { + run := func(t *testing.T, frees, lives [3]int, want string) { + t.Helper() + topo := NewTopology() + ids := [3]string{"n0:8080", "n1:8081", "n2:8082"} + for i, id := range ids { + n := topo.AddNode(id, "dc1", "dc1:rack0", frees[i]) + n.AddDisk(0, "", frees[i], lives[i]) + } + if got := placeFirstShardNode(t, topo, 1); got != want { + t.Fatalf("got %s, want %s (110 in-band, not 100/120)", got, want) + } + } + t.Run("high-to-low", func(t *testing.T) { + run(t, [3]int{120, 110, 100}, [3]int{300, 200, 100}, "n1:8081") + }) + t.Run("low-to-high", func(t *testing.T) { + run(t, [3]int{100, 110, 120}, [3]int{100, 200, 300}, "n1:8081") + }) +} + +func TestRackDeadbandAnchorsOnMaxFree(t *testing.T) { + run := func(t *testing.T, frees, lives [3]int, want string) { + t.Helper() + topo := NewTopology() + for i := 0; i < 3; i++ { + n := topo.AddNode(fmt.Sprintf("n%d:8080", i), "dc1", fmt.Sprintf("dc1:rack%d", i), frees[i]) + n.AddDisk(0, "", frees[i], lives[i]) + } + res, err := topo.Place(1, "c1", []int{0}, Constraints{FreeSlotsDeadband: 1}, PlaceStrict) + if err != nil { + t.Fatalf("Place: %v", err) + } + d, ok := res.Destinations[0] + if !ok { + t.Fatal("missing shard 0") + } + if d.Rack != want { + t.Fatalf("got rack %s, want %s (110 in-band, not 100/120)", d.Rack, want) + } + } + t.Run("high-to-low", func(t *testing.T) { + run(t, [3]int{120, 110, 100}, [3]int{300, 200, 100}, "rack1") + }) + t.Run("low-to-high", func(t *testing.T) { + run(t, [3]int{100, 110, 120}, [3]int{100, 200, 300}, "rack1") + }) +} diff --git a/weed/worker/tasks/erasure_coding/config.go b/weed/worker/tasks/erasure_coding/config.go index b356a61b08..e1eba8fb03 100644 --- a/weed/worker/tasks/erasure_coding/config.go +++ b/weed/worker/tasks/erasure_coding/config.go @@ -12,13 +12,14 @@ import ( // Config extends BaseConfig with erasure coding specific settings type Config struct { base.BaseConfig - QuietForSeconds int `json:"quiet_for_seconds"` - FullnessRatio float64 `json:"fullness_ratio"` - CollectionFilter string `json:"collection_filter"` - MinSizeMB int `json:"min_size_mb"` - PreferredTags []string `json:"preferred_tags"` - ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication - StrictPlacement bool `json:"strict_placement"` // fail planning instead of relaxing placement constraints + QuietForSeconds int `json:"quiet_for_seconds"` + FullnessRatio float64 `json:"fullness_ratio"` + CollectionFilter string `json:"collection_filter"` + MinSizeMB int `json:"min_size_mb"` + PreferredTags []string `json:"preferred_tags"` + ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication + StrictPlacement bool `json:"strict_placement"` // fail planning instead of relaxing placement constraints + FreeSlotsDeadband int `json:"free_slots_deadband"` // Place volume-slot M; |ΔfreeSlots| ≤ M×dataShards is a tie, then liveShards; 0 = strict } // NewDefaultConfig creates a new default erasure coding configuration @@ -184,6 +185,22 @@ func GetConfigSpec() base.ConfigSpec { InputType: "checkbox", CSSClasses: "form-check-input", }, + { + Name: "free_slots_deadband", + JSONName: "free_slots_deadband", + Type: config.FieldTypeInt, + DefaultValue: 0, + MinValue: 0, + MaxValue: 100000, + Required: false, + DisplayName: "Free Volume Deadband (M)", + Description: "Treat |Δfree volume slots| ≤ M as a Place tie and rank by liveShards instead", + HelpText: "0 (default) keeps strict freeSlots ranking. A positive M is volume slots; Place applies M × data shards (10+4 → ×10).", + Placeholder: "0", + Unit: config.UnitCount, + InputType: "number", + CSSClasses: "form-control", + }, }, } } @@ -199,13 +216,14 @@ func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy { CheckIntervalSeconds: int32(c.ScanIntervalSeconds), TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{ ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{ - FullnessRatio: float64(c.FullnessRatio), - QuietForSeconds: int32(c.QuietForSeconds), - MinVolumeSizeMb: int32(c.MinSizeMB), - CollectionFilter: c.CollectionFilter, - PreferredTags: preferredTagsCopy, - ReplicaPlacement: c.ReplicaPlacement, - StrictPlacement: c.StrictPlacement, + FullnessRatio: float64(c.FullnessRatio), + QuietForSeconds: int32(c.QuietForSeconds), + MinVolumeSizeMb: int32(c.MinSizeMB), + CollectionFilter: c.CollectionFilter, + PreferredTags: preferredTagsCopy, + ReplicaPlacement: c.ReplicaPlacement, + StrictPlacement: c.StrictPlacement, + FreeSlotsDeadband: int32(c.FreeSlotsDeadband), }, }, } @@ -231,6 +249,7 @@ func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error { c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...) c.ReplicaPlacement = ecConfig.ReplicaPlacement c.StrictPlacement = ecConfig.StrictPlacement + c.FreeSlotsDeadband = int(ecConfig.FreeSlotsDeadband) } return nil diff --git a/weed/worker/tasks/erasure_coding/detection.go b/weed/worker/tasks/erasure_coding/detection.go index de48d6c55d..a422964ef2 100644 --- a/weed/worker/tasks/erasure_coding/detection.go +++ b/weed/worker/tasks/erasure_coding/detection.go @@ -519,11 +519,12 @@ func planECDestinations(snap *ecbalancer.Topology, nodeAddresses map[string]stri mode = ecbalancer.PlaceStrict } res, err := snap.Place(metric.VolumeID, metric.Collection, need, ecbalancer.Constraints{ - DiskType: metric.DiskType, - DiskTypePolicy: ecbalancer.DiskTypePrefer, - PreferredTags: ecConfig.PreferredTags, - ReplicaPlacement: rp, - Ratio: func(string) (int, int) { return dataShards, parityShards }, + DiskType: metric.DiskType, + DiskTypePolicy: ecbalancer.DiskTypePrefer, + PreferredTags: ecConfig.PreferredTags, + ReplicaPlacement: rp, + Ratio: func(string) (int, int) { return dataShards, parityShards }, + FreeSlotsDeadband: ecConfig.FreeSlotsDeadband, }, mode) if err != nil { return nil, nil, err diff --git a/weed/worker/tasks/erasure_coding/detection_strict_test.go b/weed/worker/tasks/erasure_coding/detection_strict_test.go index 7b7cddbb61..8916c3d960 100644 --- a/weed/worker/tasks/erasure_coding/detection_strict_test.go +++ b/weed/worker/tasks/erasure_coding/detection_strict_test.go @@ -39,12 +39,16 @@ func TestPlanECDestinationsStrictPlacement(t *testing.T) { func TestStrictPlacementRoundTripsThroughTaskPolicy(t *testing.T) { cfg := NewDefaultConfig() cfg.StrictPlacement = true + cfg.FreeSlotsDeadband = 3 restored := NewDefaultConfig() require.NoError(t, restored.FromTaskPolicy(cfg.ToTaskPolicy())) require.True(t, restored.StrictPlacement, "strict placement must survive the persisted policy round trip") + require.Equal(t, 3, restored.FreeSlotsDeadband, "free_slots_deadband must survive the persisted policy round trip") restored.StrictPlacement = false + restored.FreeSlotsDeadband = 0 require.NoError(t, restored.FromTaskPolicy(restored.ToTaskPolicy())) require.False(t, restored.StrictPlacement) + require.Zero(t, restored.FreeSlotsDeadband) } diff --git a/weed/worker/tasks/erasure_coding/plugin_handler.go b/weed/worker/tasks/erasure_coding/plugin_handler.go index 36ffd5a59e..5071529b6d 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler.go @@ -98,7 +98,7 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { WorkerConfigForm: &plugin_pb.ConfigForm{ FormId: "erasure-coding-worker", Title: "Erasure Coding Worker Config", - Description: "Worker-side detection thresholds.", + Description: "Worker-side detection and shard placement.", Sections: []*plugin_pb.ConfigSection{ { SectionId: "thresholds", @@ -133,6 +133,13 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { Required: true, MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 1}}, }, + }, + }, + { + SectionId: "placement", + Title: "Shard Placement", + Description: "Where EC shards go: disk tags, replica-placement caps, Place ranking.", + Fields: []*plugin_pb.ConfigField{ { Name: "preferred_tags", Label: "Preferred Tags", @@ -149,6 +156,15 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_STRING, Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_TEXT, }, + { + Name: "free_slots_deadband", + Label: "Free Volume Deadband (M)", + Description: "Nodes whose free volume slots differ by at most M are a tie; then fewer live shards wins. Applied as M × data shards (10+4 → ×10). 0 = always pick more free slots.", + FieldType: plugin_pb.ConfigFieldType_CONFIG_FIELD_TYPE_INT64, + Widget: plugin_pb.ConfigWidget_CONFIG_WIDGET_NUMBER, + MinValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}}, + MaxValue: &plugin_pb.ConfigValue{Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 100000}}, + }, { Name: "strict_placement", Label: "Strict Placement", @@ -178,6 +194,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { "strict_placement": { Kind: &plugin_pb.ConfigValue_BoolValue{BoolValue: false}, }, + "free_slots_deadband": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}, + }, }, }, AdminRuntimeDefaults: &plugin_pb.AdminRuntimeDefaults{ @@ -205,6 +224,9 @@ func (h *ErasureCodingHandler) Descriptor() *plugin_pb.JobTypeDescriptor { "preferred_tags": { Kind: &plugin_pb.ConfigValue_StringValue{StringValue: ""}, }, + "free_slots_deadband": { + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 0}, + }, }, } } @@ -640,6 +662,14 @@ func deriveErasureCodingWorkerConfig(values map[string]*plugin_pb.ConfigValue) * taskConfig.StrictPlacement = pluginworker.ReadBoolConfig(values, "strict_placement", taskConfig.StrictPlacement) + deadband := pluginworker.ReadIntConfig(values, "free_slots_deadband", taskConfig.FreeSlotsDeadband) + if deadband < 0 { + deadband = 0 + } else if deadband > 100000 { + deadband = 100000 + } + taskConfig.FreeSlotsDeadband = deadband + return &erasureCodingWorkerConfig{ TaskConfig: taskConfig, } diff --git a/weed/worker/tasks/erasure_coding/plugin_handler_test.go b/weed/worker/tasks/erasure_coding/plugin_handler_test.go index 6febba84b9..71e221ce4d 100644 --- a/weed/worker/tasks/erasure_coding/plugin_handler_test.go +++ b/weed/worker/tasks/erasure_coding/plugin_handler_test.go @@ -128,6 +128,17 @@ func TestDeriveErasureCodingWorkerConfig(t *testing.T) { if cfg.TaskConfig.MinSizeMB != 128 { t.Fatalf("expected min_size_mb 128, got %d", cfg.TaskConfig.MinSizeMB) } + if cfg.TaskConfig.FreeSlotsDeadband != 0 { + t.Fatalf("expected default free_slots_deadband 0, got %d", cfg.TaskConfig.FreeSlotsDeadband) + } + + values["free_slots_deadband"] = &plugin_pb.ConfigValue{ + Kind: &plugin_pb.ConfigValue_Int64Value{Int64Value: 12}, + } + cfg = deriveErasureCodingWorkerConfig(values) + if cfg.TaskConfig.FreeSlotsDeadband != 12 { + t.Fatalf("expected free_slots_deadband 12, got %d", cfg.TaskConfig.FreeSlotsDeadband) + } } func TestBuildErasureCodingProposal(t *testing.T) { @@ -249,6 +260,51 @@ func TestEmitErasureCodingDetectionDecisionTraceNoTasks(t *testing.T) { } } +func TestErasureCodingDescriptorExposesFreeSlotsDeadband(t *testing.T) { + descriptor := NewErasureCodingHandler(nil, "").Descriptor() + if descriptor == nil || descriptor.WorkerConfigForm == nil { + t.Fatalf("expected worker config form in descriptor") + } + if descriptor.DescriptorVersion != 1 { + t.Fatalf("descriptor version = %d, want 1", descriptor.DescriptorVersion) + } + if !workerConfigFormHasField(descriptor.WorkerConfigForm, "free_slots_deadband") { + t.Fatal("worker config form omitted free_slots_deadband") + } + var sectionID, prev string + for _, section := range descriptor.WorkerConfigForm.Sections { + if section == nil { + continue + } + for _, field := range section.Fields { + if field != nil && field.Name == "free_slots_deadband" { + sectionID = section.SectionId + if field.Label != "Free Volume Deadband (M)" { + t.Fatalf("free_slots_deadband label = %q, want Free Volume Deadband (M)", field.Label) + } + } + if field != nil && field.Name == "replica_placement" { + prev = section.SectionId + } + } + } + if sectionID != "placement" { + t.Fatalf("free_slots_deadband section = %q, want placement", sectionID) + } + if prev != "placement" { + t.Fatalf("replica_placement section = %q, want placement next to free_slots_deadband", prev) + } + for _, section := range descriptor.WorkerConfigForm.Sections { + if section != nil && section.SectionId == "thresholds" { + for _, field := range section.Fields { + if field != nil && field.Name == "free_slots_deadband" { + t.Fatal("free_slots_deadband must not sit under Detection Thresholds") + } + } + } + } +} + func TestErasureCodingDescriptorOmitsLocalExecutionFields(t *testing.T) { descriptor := NewErasureCodingHandler(nil, "").Descriptor() if descriptor == nil || descriptor.WorkerConfigForm == nil { From 34c7bd03add0394c9911d2f1c77b038e70fa9e65 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 07:07:55 -0700 Subject: [PATCH 65/70] ci: clean up stale FUSE mounts before running tests (#12020) * ci(s3tables): tolerate stale FUSE mounts in disk diagnostics * ci(s3tables): bound df diagnostics with timeout, explain once A stale FUSE mount can also hang statfs, not just fail it, so wrap df in timeout 15; one comment at the first site covers all sixteen. * ci: tolerate stale FUSE mounts in df diagnostics elsewhere The same self-hosted runners serve the other workflow files; a bare df -h there can stall or fail a job the same way. * ci: unmount disconnected FUSE test mounts before jobs start * ci: skip FUSE cleanup when the variant checkout is skipped * ci: reclaim the crashed run's test dir once its stale FUSE mount is detached * ci: stop tracking python bytecode for the FUSE cleanup action The helper's __pycache__ slipped into the tree; ignore it so local test runs cannot re-add it. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: run the FUSE cleanup action's unit tests on action changes test_cleanup.py was never wired into CI, so a regression in cleanup.py would ship silently. A path-filtered workflow runs it on every change under .github/actions/. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci: match every FUSE test mount layout in the cleanup action The candidates list only matched seaweedfs_fuse_test_*/mount, so stale mounts left by fuse_failover and fuse_p2p runs (mountN leaves under seaweedfs_fuse_* dirs) and the posix-compliance workflow's fixed /tmp/seaweedfs/mount were never reclaimed and could still wedge df. * ci: run the FUSE mount cleanup in the fuse_failover and fuse_p2p workflows --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../cleanup-stale-fuse-mounts/action.yml | 11 ++ .../cleanup-stale-fuse-mounts/cleanup.py | 89 ++++++++++++ .../cleanup-stale-fuse-mounts/test_cleanup.py | 137 ++++++++++++++++++ .github/workflows/action-tests.yml | 30 ++++ .github/workflows/container_latest.yml | 7 +- .../workflows/container_release_unified.yml | 6 +- .github/workflows/fuse-failover.yml | 3 + .github/workflows/fuse-p2p-integration.yml | 3 + .github/workflows/posix-compliance.yml | 5 +- .github/workflows/s3-filer-group-tests.yml | 5 +- .github/workflows/s3-go-tests.yml | 20 ++- .github/workflows/s3-iam-tests.yml | 5 +- .github/workflows/s3-keycloak-tests.yml | 5 +- .github/workflows/s3-policy-tests.yml | 10 +- .github/workflows/s3-spark-tests.yml | 5 +- .github/workflows/s3-sse-tests.yml | 5 +- .github/workflows/s3-tables-tests.yml | 81 +++++++++-- .github/workflows/tus-tests.yml | 5 +- .gitignore | 1 + 19 files changed, 401 insertions(+), 32 deletions(-) create mode 100644 .github/actions/cleanup-stale-fuse-mounts/action.yml create mode 100644 .github/actions/cleanup-stale-fuse-mounts/cleanup.py create mode 100644 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py create mode 100644 .github/workflows/action-tests.yml diff --git a/.github/actions/cleanup-stale-fuse-mounts/action.yml b/.github/actions/cleanup-stale-fuse-mounts/action.yml new file mode 100644 index 0000000000..7e6ddab7bd --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/action.yml @@ -0,0 +1,11 @@ +name: Clean up stale SeaweedFS test mounts +description: Detach disconnected FUSE test mounts left by earlier jobs on a shared runner. +runs: + using: composite + steps: + - name: Unmount disconnected SeaweedFS test mounts + if: runner.os == 'Linux' && runner.environment == 'self-hosted' + shell: bash + env: + CLEANUP_SCRIPT: ${{ github.action_path }}/cleanup.py + run: python3 "$CLEANUP_SCRIPT" diff --git a/.github/actions/cleanup-stale-fuse-mounts/cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py new file mode 100644 index 0000000000..ad17a767fa --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/cleanup.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Detach disconnected SeaweedFS integration-test mounts without touching live ones.""" + +import errno +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile + + +def test_mounts(mountinfo): + """Read the kernel table directly: traversing a stale mount may block.""" + roots = {tempfile.gettempdir(), "/tmp", "/var/tmp"} + for line in mountinfo.splitlines(): + fields, filesystem = line.split(" - ", 1) + if filesystem.split()[0].split(".")[0] != "fuse": + continue + fields = fields.split() + # mountinfo escapes whitespace and backslashes with octal sequences. + target = re.sub(r"\\([0-7]{3})", lambda m: chr(int(m[1], 8)), fields[4]) + path = Path(target) + # Test mounts are a mount* leaf under a seaweedfs_fuse_* scratch dir + # (seaweedfs_fuse_test_*/mount, seaweedfs_fuse_failover_*/mount0, ...) + # or the posix-compliance workflow's fixed /tmp/seaweedfs/mount. + if (not path.name.startswith("mount") + or str(path.parent.parent) not in roots + or not (path.parent.name.startswith("seaweedfs_fuse") + or path.parent.name == "seaweedfs")): + continue + yield fields[0], target + + +def mounted(mount_id, target): + return (mount_id, target) in test_mounts(Path("/proc/self/mountinfo").read_text()) + + +def cleanup(): + failed = False + for mount_id, target in test_mounts(Path("/proc/self/mountinfo").read_text()): + try: + # A separate, bounded probe prevents a wedged FUSE daemon from + # hanging cleanup. Only ENOTCONN proves the daemon disconnected; + # a timeout or another error is not enough to detach a live mount. + probe = subprocess.run( + [sys.executable, __file__, "--probe", target], timeout=5, + capture_output=True, check=False, + ) + except subprocess.TimeoutExpired: + print(f"::warning::FUSE probe timed out; leaving {target} mounted") + continue + if probe.returncode != errno.ENOTCONN or not mounted(mount_id, target): + continue + + print(f"Unmounting disconnected FUSE test mount: {target}", flush=True) + # Lazy unmount detaches a dead mount even when an old process still + # holds it open. Prefer the unprivileged FUSE helper; sudo is fallback. + for command in ( + ["fusermount3", "-uz", "--", target], + ["fusermount", "-uz", "--", target], + ["sudo", "-n", "umount", "-l", "--", target], + ): + try: + subprocess.run(command, timeout=10, check=False) + except (OSError, subprocess.TimeoutExpired) as error: + print(f"{command[0]}: {error}", file=sys.stderr) + if not mounted(mount_id, target): + break + else: + print(f"::error::Could not unmount disconnected FUSE test mount: {target}") + failed = True + continue + # The detached mount's parent still holds the crashed run's data and + # logs; test_mounts only yields targets under a SeaweedFS test dir in + # a temp root, so the parent is safe to reclaim here. + shutil.rmtree(Path(target).parent, ignore_errors=True) + return int(failed) + + +if __name__ == "__main__": + if len(sys.argv) == 3 and sys.argv[1] == "--probe": + try: + os.stat(sys.argv[2]) + except OSError as error: + sys.exit(error.errno) + sys.exit(0) + sys.exit(cleanup()) diff --git a/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py new file mode 100644 index 0000000000..b8ee3fa028 --- /dev/null +++ b/.github/actions/cleanup-stale-fuse-mounts/test_cleanup.py @@ -0,0 +1,137 @@ +import errno +import importlib.util +from pathlib import Path +import subprocess +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location("cleanup", Path(__file__).with_name("cleanup.py")) +cleanup = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cleanup) + +TARGET = "/tmp/seaweedfs_fuse_test_123/mount" +MOUNTINFO = f"42 1 0:9 / {TARGET} rw - fuse.seaweedfs seaweedfs rw\n" + + +class CleanupTests(unittest.TestCase): + def run_cleanup(self, results, mounted=True): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO), \ + patch.object(cleanup, "mounted", side_effect=mounted if isinstance(mounted, list) else None, + return_value=mounted) as check, \ + patch.object(cleanup.shutil, "rmtree") as rmtree, \ + patch.object(cleanup.subprocess, "run", side_effect=results) as run: + status = cleanup.cleanup() + return status, run, check, rmtree + + def result(self, status): + return subprocess.CompletedProcess([], status) + + def test_only_fuse_integration_test_mounts_are_candidates(self): + rows = MOUNTINFO + rows += MOUNTINFO.replace("fuse.seaweedfs", "ext4") + rows += MOUNTINFO.replace(TARGET, "/mnt/production") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs_fuse_test_123/data") + rows += MOUNTINFO.replace(TARGET, "/home/user/seaweedfs_fuse_test_123/mount") + rows += MOUNTINFO.replace(TARGET, "/tmp/seaweedfs/data") + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", TARGET)]) + + def test_all_test_mount_layouts_are_candidates(self): + targets = ( + "/tmp/seaweedfs_fuse_test_123/mount", + "/tmp/seaweedfs_fuse_failover_456/mount0", + "/tmp/seaweedfs_fuse_failover_456/mount1", + "/var/tmp/seaweedfs_fuse_p2p_test_789/mount2", + "/tmp/seaweedfs/mount", + ) + rows = "".join( + MOUNTINFO.replace("42", str(40 + i)).replace(TARGET, target) + for i, target in enumerate(targets) + ) + self.assertEqual( + list(cleanup.test_mounts(rows)), + [(str(40 + i), target) for i, target in enumerate(targets)], + ) + + def test_mountinfo_paths_are_unescaped_once(self): + target = "/tmp/seaweedfs_fuse_test_space and\\slash/mount" + encoded = target.replace("\\", r"\134").replace(" ", r"\040") + rows = MOUNTINFO.replace(TARGET, encoded) + self.assertEqual(list(cleanup.test_mounts(rows)), [("42", target)]) + + def test_no_mounts_is_a_noop(self): + with patch.object(cleanup.Path, "read_text", return_value=""), \ + patch.object(cleanup.subprocess, "run") as run: + self.assertEqual(cleanup.cleanup(), 0) + run.assert_not_called() + + def test_healthy_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(0)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_other_probe_errors_are_untouched(self): + for error in (errno.EACCES, errno.ENOENT, errno.EIO): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup([self.result(error)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_probe_timeout_does_not_detach_a_possibly_live_mount(self): + status, run, _, _ = self.run_cleanup([subprocess.TimeoutExpired("probe", 5)]) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_disconnected_mount_is_lazily_unmounted(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args_list[1].args[0], ["fusermount3", "-uz", "--", TARGET]) + self.assertEqual(run.call_args_list[0].kwargs["timeout"], 5) + self.assertEqual(run.call_args_list[1].kwargs["timeout"], 10) + + def test_replaced_or_removed_mount_is_untouched(self): + status, run, _, _ = self.run_cleanup([self.result(errno.ENOTCONN)], False) + self.assertEqual(status, 0) + self.assertEqual(run.call_count, 1) + + def test_mount_identity_is_checked_without_stat(self): + with patch.object(cleanup.Path, "read_text", return_value=MOUNTINFO): + self.assertTrue(cleanup.mounted("42", TARGET)) + self.assertFalse(cleanup.mounted("43", TARGET)) + + def test_missing_or_timed_out_helper_uses_fallback(self): + for error in (FileNotFoundError(), subprocess.TimeoutExpired("fusermount3", 10)): + with self.subTest(error=error): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), error, self.result(0)], [True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["fusermount", "-uz", "--", TARGET]) + + def test_failed_helpers_fall_back_to_sudo(self): + status, run, _, _ = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(0)], + [True, True, True, False]) + self.assertEqual(status, 0) + self.assertEqual(run.call_args.args[0], ["sudo", "-n", "umount", "-l", "--", TARGET]) + + def test_cleanup_fails_when_stale_mount_remains(self): + status, run, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(1), self.result(1), self.result(1)]) + self.assertEqual(status, 1) + self.assertEqual(run.call_count, 4) + rmtree.assert_not_called() + + def test_unmounted_test_dir_is_reclaimed(self): + status, _, _, rmtree = self.run_cleanup( + [self.result(errno.ENOTCONN), self.result(0)], [True, False]) + self.assertEqual(status, 0) + rmtree.assert_called_once_with(Path(TARGET).parent, ignore_errors=True) + + def test_probe_process_reports_errno(self): + for target, expected in ((str(Path(__file__)), 0), (str(Path(__file__) / "absent"), errno.ENOTDIR)): + with self.subTest(target=target): + result = subprocess.run([cleanup.sys.executable, cleanup.__file__, "--probe", target], check=False) + self.assertEqual(result.returncode, expected) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/action-tests.yml b/.github/workflows/action-tests.yml new file mode 100644 index 0000000000..88f33ed93c --- /dev/null +++ b/.github/workflows/action-tests.yml @@ -0,0 +1,30 @@ +name: "actions: unit tests" + +on: + push: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + pull_request: + branches: [ master ] + paths: + - '.github/actions/**' + - '.github/workflows/action-tests.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || (github.event_name == 'push' && github.ref) || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) }} + +permissions: + contents: read + +jobs: + cleanup-stale-fuse-mounts: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v7 + + - name: Run cleanup action unit tests + run: python3 .github/actions/cleanup-stale-fuse-mounts/test_cleanup.py diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index 0dc49db29c..8eccfc89d4 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -170,10 +170,13 @@ jobs: uses: actions/checkout@v7 with: ref: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.source_ref || github.ref }} + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Free Disk Space run: | echo "Available disk space before cleanup:" - df -h + timeout 15 df -h || true # Remove pre-installed tools sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL # Clean package managers @@ -185,7 +188,7 @@ jobs: [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true echo "Available disk space after cleanup:" - df -h + timeout 15 df -h || true - name: Configure variant id: config diff --git a/.github/workflows/container_release_unified.yml b/.github/workflows/container_release_unified.yml index 320436cb2e..17d976509e 100644 --- a/.github/workflows/container_release_unified.yml +++ b/.github/workflows/container_release_unified.yml @@ -180,6 +180,10 @@ jobs: if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Download pre-built Rust binaries if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant uses: actions/download-artifact@v8 @@ -219,7 +223,7 @@ jobs: sudo docker system prune -af --volumes [ -d ~/.cache/go-build ] && rm -rf ~/.cache/go-build || true [ -d /go/pkg ] && rm -rf /go/pkg || true - df -h + timeout 15 df -h || true - name: Docker meta if: github.event_name != 'workflow_dispatch' || github.event.inputs.variant == 'all' || github.event.inputs.variant == matrix.variant diff --git a/.github/workflows/fuse-failover.yml b/.github/workflows/fuse-failover.yml index 21646dbe17..2c03cf9369 100644 --- a/.github/workflows/fuse-failover.yml +++ b/.github/workflows/fuse-failover.yml @@ -42,6 +42,9 @@ jobs: with: persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: diff --git a/.github/workflows/fuse-p2p-integration.yml b/.github/workflows/fuse-p2p-integration.yml index 78df490b43..a65f5ddbc2 100644 --- a/.github/workflows/fuse-p2p-integration.yml +++ b/.github/workflows/fuse-p2p-integration.yml @@ -41,6 +41,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: diff --git a/.github/workflows/posix-compliance.yml b/.github/workflows/posix-compliance.yml index 25fbfe5096..66efc0de44 100644 --- a/.github/workflows/posix-compliance.yml +++ b/.github/workflows/posix-compliance.yml @@ -58,6 +58,9 @@ jobs: with: fetch-depth: 1 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v7 with: @@ -393,7 +396,7 @@ jobs: echo "" >> reports/debug/system_info.txt echo "Disk Space" >> reports/debug/system_info.txt - df -h >> reports/debug/system_info.txt + timeout 15 df -h >> reports/debug/system_info.txt || true echo "" >> reports/debug/system_info.txt echo "Memory Information" >> reports/debug/system_info.txt diff --git a/.github/workflows/s3-filer-group-tests.yml b/.github/workflows/s3-filer-group-tests.yml index 18f64fd78c..d714ca98e3 100644 --- a/.github/workflows/s3-filer-group-tests.yml +++ b/.github/workflows/s3-filer-group-tests.yml @@ -32,6 +32,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -51,7 +54,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Filer Group Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-go-tests.yml b/.github/workflows/s3-go-tests.yml index 7660ce1d71..1260e5af41 100644 --- a/.github/workflows/s3-go-tests.yml +++ b/.github/workflows/s3-go-tests.yml @@ -35,6 +35,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -54,7 +57,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -201,6 +204,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -220,7 +226,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management @@ -339,6 +345,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -358,7 +367,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" make test-with-server @@ -398,6 +407,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -417,7 +429,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h . /tmp + timeout 15 df -h || true echo "=== Starting Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-iam-tests.yml b/.github/workflows/s3-iam-tests.yml index ee12afd842..9d57baf978 100644 --- a/.github/workflows/s3-iam-tests.yml +++ b/.github/workflows/s3-iam-tests.yml @@ -91,6 +91,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -111,7 +114,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 IAM Integration Tests (${{ matrix.test-type }}) ===" # Set WEED_BINARY to use the installed version diff --git a/.github/workflows/s3-keycloak-tests.yml b/.github/workflows/s3-keycloak-tests.yml index d893b1063b..f3955ca3fc 100644 --- a/.github/workflows/s3-keycloak-tests.yml +++ b/.github/workflows/s3-keycloak-tests.yml @@ -37,6 +37,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -57,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Keycloak Integration Tests ===" # Set WEED_BINARY to use the installed version diff --git a/.github/workflows/s3-policy-tests.yml b/.github/workflows/s3-policy-tests.yml index caa9f9de6f..2fbeff3474 100644 --- a/.github/workflows/s3-policy-tests.yml +++ b/.github/workflows/s3-policy-tests.yml @@ -89,6 +89,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -108,7 +111,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Variables Integration Tests ===" @@ -199,6 +202,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -218,7 +224,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Policy Enforcement Tests (${{ matrix.test-case }}) ===" diff --git a/.github/workflows/s3-spark-tests.yml b/.github/workflows/s3-spark-tests.yml index 39ce8d9991..59fae691a8 100644 --- a/.github/workflows/s3-spark-tests.yml +++ b/.github/workflows/s3-spark-tests.yml @@ -27,6 +27,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -57,7 +60,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting S3 Spark Integration Tests ===" go test -v -timeout 30m . 2>&1 | tee test-output.log || { diff --git a/.github/workflows/s3-sse-tests.yml b/.github/workflows/s3-sse-tests.yml index 4c44e42448..2ea0531f0a 100644 --- a/.github/workflows/s3-sse-tests.yml +++ b/.github/workflows/s3-sse-tests.yml @@ -46,6 +46,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -65,7 +68,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting SSE Tests ===" # Run tests with automatic server management diff --git a/.github/workflows/s3-tables-tests.yml b/.github/workflows/s3-tables-tests.yml index baa50a4cb9..19674a3325 100644 --- a/.github/workflows/s3-tables-tests.yml +++ b/.github/workflows/s3-tables-tests.yml @@ -30,6 +30,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -53,7 +56,8 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + # A stale FUSE mount can hang or fail df; keep diagnostics best effort. + timeout 15 df -h || true echo "=== Starting S3 Tables Tests ===" # Run S3 Tables integration tests @@ -91,6 +95,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -115,7 +122,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Iceberg Catalog Tests ===" # Run Iceberg catalog integration tests using Makefile (handles build) @@ -153,6 +160,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -186,7 +196,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Trino Iceberg Catalog Tests ===" # Run Trino + Iceberg catalog integration tests @@ -224,6 +234,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -258,7 +271,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Dremio Iceberg Catalog Tests ===" @@ -298,6 +311,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -332,7 +348,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Doris Iceberg Catalog Tests ===" @@ -382,6 +398,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -418,7 +437,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting ClickHouse Iceberg Catalog Tests ===" @@ -470,6 +489,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -507,7 +529,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting OLake Iceberg Catalog Tests ===" @@ -570,6 +592,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -602,7 +627,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Polaris Tests ===" go test -v -timeout 20m ./test/s3tables/polaris 2>&1 | tee test/s3tables/polaris/test-output.log || { @@ -639,6 +664,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -672,7 +700,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Spark Iceberg Catalog Tests ===" # Run Spark + Iceberg catalog integration tests @@ -710,6 +738,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -744,7 +775,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting RisingWave Catalog Tests ===" # Run RisingWave catalog integration tests @@ -782,6 +813,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -840,7 +874,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting STS Integration Tests ===" # Run STS integration tests @@ -878,6 +912,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -901,7 +938,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting Lakekeeper Integration Tests ===" # Run Lakekeeper integration tests @@ -939,6 +976,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -973,7 +1013,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Unity Catalog Tests ===" @@ -1013,6 +1053,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1046,7 +1089,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting LanceDB Namespace Tests ===" @@ -1157,6 +1200,9 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1191,7 +1237,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting DuckDB Lance Tests ===" @@ -1233,6 +1279,9 @@ jobs: # so do not leave a token in the checkout for it to pick up. persist-credentials: false + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -1266,7 +1315,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true docker info echo "=== Starting Spark Lance Namespace Tests ===" diff --git a/.github/workflows/tus-tests.yml b/.github/workflows/tus-tests.yml index 699c36b0b5..1140c7f505 100644 --- a/.github/workflows/tus-tests.yml +++ b/.github/workflows/tus-tests.yml @@ -31,6 +31,9 @@ jobs: - name: Check out code uses: actions/checkout@v7 + - name: Clean up stale FUSE mounts + uses: ./.github/actions/cleanup-stale-fuse-mounts + - name: Set up Go uses: actions/setup-go@v7 with: @@ -50,7 +53,7 @@ jobs: echo "=== System Information ===" uname -a free -h - df -h + timeout 15 df -h || true echo "=== Starting TUS Tests ===" # Run tests with automatic server management diff --git a/.gitignore b/.gitignore index 9a3bf80cd3..c383623df9 100644 --- a/.gitignore +++ b/.gitignore @@ -144,3 +144,4 @@ weed_bin telemetry/server/telemetry-server .aider* /seaweed-volume/docs +__pycache__/ From fcd28031e7f2818fc7347003a420a099df963e12 Mon Sep 17 00:00:00 2001 From: Ping Qiu Date: Sat, 24 Oct 2026 07:09:43 -0700 Subject: [PATCH 66/70] mq/kafka: wait for a filer to register instead of panicking on an empty list (#12019) * mq/kafka: wait for a filer to register instead of panicking on an empty list NewSeaweedMQBrokerHandler queried the masters for filers once, ~2 s after start, and handed an empty list to NewFilerClientAccessor, which panics. A fresh filer registers only after its empty-membership confirm window (10 s), so a gateway started with a fresh cluster crashed (Kafka Quick Test, kafka-gateway exit 2). Retry while the list is empty for up to 60 s, then return an error. Co-Authored-By: Claude Fable 5.1 * mq/kafka: wait out a transient-empty broker listing too A fresh SMQ broker registers with the masters on the same announce delay as a filer, so the one-shot broker lookup could fail startup the same way. Generalize waitForFilers into waitForClusterNodes and bound both discoveries by the same window; the empty-broker check becomes unreachable and drops away. * mq/kafka: wait out the broker health check like node discovery A broker announces to the masters before it has acquired its balancer lock, so the gateway's three-attempt probe could still fail fast on a fresh cluster. Retry FindBrokerLeader for the same window the caller waits out empty listings. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/mq/kafka/integration/broker_client.go | 19 ++++---- .../kafka/integration/broker_client_test.go | 46 +++++++++++++++++++ .../integration/seaweedmq_handler_utils.go | 44 ++++++++++++++---- .../seaweedmq_handler_utils_wait_test.go | 38 +++++++++++++++ 4 files changed, 130 insertions(+), 17 deletions(-) create mode 100644 weed/mq/kafka/integration/broker_client_test.go create mode 100644 weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go diff --git a/weed/mq/kafka/integration/broker_client.go b/weed/mq/kafka/integration/broker_client.go index 3c3482c86e..428cfdffef 100644 --- a/weed/mq/kafka/integration/broker_client.go +++ b/weed/mq/kafka/integration/broker_client.go @@ -22,9 +22,8 @@ import ( ) const ( - brokerHealthCheckTimeout = 5 * time.Second - brokerHealthCheckAttempts = 3 - brokerHealthCheckBackoff = 250 * time.Millisecond + brokerHealthCheckTimeout = 5 * time.Second + brokerHealthCheckBackoff = 250 * time.Millisecond ) // NewBrokerClientWithFilerAccessor creates a client with a shared filer accessor @@ -102,14 +101,17 @@ func (bc *BrokerClient) Close() error { return bc.conn.Close() } -// HealthCheck verifies the broker connection is working +// HealthCheck verifies the broker connection is working. A freshly announced +// broker may still be acquiring its balancer lock, so keep retrying for the +// same window cluster discovery waits out. func (bc *BrokerClient) HealthCheck() error { if bc.client == nil { return fmt.Errorf("broker client not connected") } + deadline := time.Now().Add(clusterNodeDiscoveryTimeout) var lastErr error - for attempt := 1; attempt <= brokerHealthCheckAttempts; attempt++ { + for attempt := 1; ; attempt++ { ctx, cancel := context.WithTimeout(bc.ctx, brokerHealthCheckTimeout) _, err := bc.client.FindBrokerLeader(ctx, &mq_pb.FindBrokerLeaderRequest{}) cancel() @@ -117,17 +119,18 @@ func (bc *BrokerClient) HealthCheck() error { return nil } lastErr = err - if attempt == brokerHealthCheckAttempts { + if !time.Now().Before(deadline) { break } + backoff := min(time.Duration(attempt)*brokerHealthCheckBackoff, time.Second) select { case <-bc.ctx.Done(): return bc.ctx.Err() - case <-time.After(time.Duration(attempt) * brokerHealthCheckBackoff): + case <-time.After(backoff): } } - return fmt.Errorf("broker health check failed after %d attempts: %v", brokerHealthCheckAttempts, lastErr) + return fmt.Errorf("broker health check failed after %v: %v", clusterNodeDiscoveryTimeout, lastErr) } // GetPartitionRangeInfo gets comprehensive range information from SeaweedMQ broker's native range manager diff --git a/weed/mq/kafka/integration/broker_client_test.go b/weed/mq/kafka/integration/broker_client_test.go new file mode 100644 index 0000000000..efcb8c9153 --- /dev/null +++ b/weed/mq/kafka/integration/broker_client_test.go @@ -0,0 +1,46 @@ +package integration + +import ( + "context" + "errors" + "testing" + "time" + + "google.golang.org/grpc" + + "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" +) + +type stubMessagingClient struct { + mq_pb.SeaweedMessagingClient + calls int + failFor int + err error +} + +func (s *stubMessagingClient) FindBrokerLeader(ctx context.Context, in *mq_pb.FindBrokerLeaderRequest, opts ...grpc.CallOption) (*mq_pb.FindBrokerLeaderResponse, error) { + s.calls++ + if s.calls <= s.failFor { + return nil, s.err + } + return &mq_pb.FindBrokerLeaderResponse{}, nil +} + +func TestHealthCheckRetriesPastTransientBrokerErrors(t *testing.T) { + stub := &stubMessagingClient{failFor: 2, err: errors.New("lock broker_balancer not found")} + bc := &BrokerClient{ctx: context.Background(), client: stub} + if err := bc.HealthCheck(); err != nil || stub.calls != 3 { + t.Fatalf("err %v after %d calls, want success on the 3rd", err, stub.calls) + } +} + +func TestHealthCheckGivesUpAtDeadline(t *testing.T) { + defer func(orig time.Duration) { clusterNodeDiscoveryTimeout = orig }(clusterNodeDiscoveryTimeout) + clusterNodeDiscoveryTimeout = 300 * time.Millisecond + + stub := &stubMessagingClient{failFor: 1 << 30, err: errors.New("lock broker_balancer not found")} + bc := &BrokerClient{ctx: context.Background(), client: stub} + if err := bc.HealthCheck(); err == nil || stub.calls < 2 { + t.Fatalf("err %v after %d calls, want repeated retries then an error", err, stub.calls) + } +} diff --git a/weed/mq/kafka/integration/seaweedmq_handler_utils.go b/weed/mq/kafka/integration/seaweedmq_handler_utils.go index 33c3590ff4..e17cf4eb69 100644 --- a/weed/mq/kafka/integration/seaweedmq_handler_utils.go +++ b/weed/mq/kafka/integration/seaweedmq_handler_utils.go @@ -52,21 +52,23 @@ func NewSeaweedMQBrokerHandler(masters string, filerGroup string, clientHost str time.Sleep(2 * time.Second) glog.V(1).Infof("Initial connection delay completed") - // Discover brokers from masters using master client + // Discover brokers and filers from masters using master client. A fresh + // node registers only after its announce delay (a filer also waits out its + // empty-membership confirm window), so an empty list right after startup + // is normal: wait rather than fail fast. glog.V(1).Infof("About to call discoverBrokersWithMasterClient...") - brokerAddresses, err := discoverBrokersWithMasterClient(ctx, masterClient, filerGroup) + brokerAddresses, err := waitForClusterNodes(ctx, "brokers", clusterNodeDiscoveryTimeout, func() ([]string, error) { + return discoverBrokersWithMasterClient(ctx, masterClient, filerGroup) + }) if err != nil { glog.Errorf("Broker discovery failed: %v", err) return nil, fmt.Errorf("failed to discover brokers: %v", err) } glog.V(1).Infof("Broker discovery returned: %v", brokerAddresses) - if len(brokerAddresses) == 0 { - return nil, fmt.Errorf("no brokers discovered from masters") - } - - // Discover filers from masters using master client - filerAddresses, err := discoverFilersWithMasterClient(ctx, masterClient, filerGroup) + filerAddresses, err := waitForClusterNodes(ctx, "filers", clusterNodeDiscoveryTimeout, func() ([]pb.ServerAddress, error) { + return discoverFilersWithMasterClient(ctx, masterClient, filerGroup) + }) if err != nil { return nil, fmt.Errorf("failed to discover filers: %v", err) } @@ -143,7 +145,31 @@ func discoverBrokersWithMasterClient(ctx context.Context, masterClient *wdclient return brokers, err } -// discoverFilersWithMasterClient queries masters for available filers using reusable master client +// clusterNodeDiscoveryTimeout bounds how long the gateway waits for the +// masters to list a node; well above a fresh node's announce delay. +var clusterNodeDiscoveryTimeout = 60 * time.Second + +// waitForClusterNodes calls discover until it returns at least one node, an +// error, or timeout passes, retrying once a second while the list is empty. +func waitForClusterNodes[T any](ctx context.Context, nodeType string, timeout time.Duration, discover func() ([]T, error)) ([]T, error) { + deadline := time.Now().Add(timeout) + for { + nodes, err := discover() + if err != nil || len(nodes) > 0 { + return nodes, err + } + if !time.Now().Before(deadline) { + return nil, fmt.Errorf("no %s registered with the masters after %v", nodeType, timeout) + } + glog.V(1).Infof("no %s registered with the masters yet; retrying", nodeType) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): + } + } +} + func discoverFilersWithMasterClient(ctx context.Context, masterClient *wdclient.MasterClient, filerGroup string) ([]pb.ServerAddress, error) { var filers []pb.ServerAddress diff --git a/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go b/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go new file mode 100644 index 0000000000..ac3c03115c --- /dev/null +++ b/weed/mq/kafka/integration/seaweedmq_handler_utils_wait_test.go @@ -0,0 +1,38 @@ +package integration + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/seaweedfs/seaweedfs/weed/pb" +) + +// A fresh filer is listed by the masters only after its announce delay, so an +// empty list must be waited out, not handed to the filer client accessor +// (which panics on it). +func TestWaitForFilersWaitsOutEmptyListing(t *testing.T) { + calls := 0 + filers, err := waitForClusterNodes(context.Background(), "filers", 10*time.Second, func() ([]pb.ServerAddress, error) { + calls++ + if calls < 3 { + return nil, nil + } + return []pb.ServerAddress{"filer:8888"}, nil + }) + if err != nil || len(filers) != 1 || calls != 3 { + t.Fatalf("filers %v err %v after %d calls, want 1 filer after 3", filers, err, calls) + } +} + +func TestWaitForFilersGivesUpWithAnError(t *testing.T) { + filers, err := waitForClusterNodes(context.Background(), "filers", 1500*time.Millisecond, func() ([]pb.ServerAddress, error) { return nil, nil }) + if err == nil || len(filers) != 0 { + t.Fatalf("still empty: filers %v err %v, want an error", filers, err) + } + boom := errors.New("master down") + if _, err := waitForClusterNodes(context.Background(), "filers", time.Minute, func() ([]pb.ServerAddress, error) { return nil, boom }); !errors.Is(err, boom) { + t.Fatalf("discovery error: got %v, want it returned at once", err) + } +} From c4d743a69fd350490de188491d74262d05e94755 Mon Sep 17 00:00:00 2001 From: Mohd Quamar Tyagi <104281681+Tyagiquamar@users.noreply.github.com> Date: Sat, 24 Oct 2026 19:48:10 +0530 Subject: [PATCH 67/70] s3: make ListObjectsV2 continuation tokens round-trip XML-unsafe keys (#11996) * s3: keep ListObjectsV2 continuation tokens round-tripping XML-unsafe keys A page ending on a key XML 1.0 cannot carry (for example one containing \x01) produced a NextContinuationToken from that raw key. The XML encoder replaced the control characters with U+FFFD, so a client resuming at the token restarted after the mangled name and skipped every key sorting between it and the real one. encoding-type=url encoded the keys but not the token, so it hit the same skip. Encode the token as base64url behind a prefix when the key is not valid XML text; keys that survive XML unchanged keep the bare key as the token, so their tokens are byte-identical to before. The handler decodes the prefix back to the raw key before listing. Fixes #11990 * s3: keep undecodable base64: list tokens resuming at the raw key * s3: round-trip ListObjectsV1 markers over XML-unsafe keys * s3: round-trip ListObjectVersions key markers over XML-unsafe keys * s3: round-trip ListMultipartUploads key markers over XML-unsafe keys * s3: decode only tokens whose key could not have stayed bare * s3: document and pin the encoded-wins rule for ambiguous list tokens A token like base64:AQ reads as both a legacy bare key and the encoded form of an XML-unsafe key. The encoded reading already won in code; state why and cover it so the choice is not reverted as a false simplification. * s3: keep request-marker echoes in the response's wire encoding The decoded marker/key-marker is a raw key; echoing it writes XML-unsafe bytes straight into the response, where the encoder turns them into U+FFFD. Re-encode each echo through encodeListContinuationToken so the client reads back the token it sent, and apply urlPathEscape for V1 when encoding-type=url is active, matching the listed keys. * s3: wire-encode the V1 NextMarker echo like the request marker * s3: pin resuming an XML-unsafe key past a sibling directory * s3: wire-encode ListMultipartUploads markers and keys under encoding-type=url Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: unescape url-encoded marker echoes on list resume Under encoding-type=url the V1 and ListMultipartUploads responses emit marker echoes through urlPathEscape, and clients pass them back verbatim. The incoming marker was used as the resume position without undoing that escaping, so resuming after a url-unsafe-but-XML-safe key like "a b" (marker "a%20b") restarted the listing at the literal escaped text. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: assert the full decoded V1 sequence in the marker round-trip test The test only checked the page count, so a regression that skipped one key while repeating another still passed. Requesting encoding-type=url keeps the XML-unsafe keys legible for the decoded comparison and exercises resuming with the escaped marker echo. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * s3: only unescape markers in the canonical urlPathEscape form A url-encoded request marker like a%41 cannot be encoder output, since urlPathEscape never emits %41 for 'A'. Unescaping it anyway resumed at aA and skipped every key sorting between a%41 and aA, including a%42. Require the unescape to re-escape to the received marker so only verbatim echoes decode; anything else stays the raw key the client sent. * s3: pin a raw-key marker that looks like an escape sequence * s3: cover the marker-echo decode guard with unit cases * s3: wire-encode the ListObjectsV2 StartAfter echo under encoding-type=url AWS encodes Delimiter, Prefix, Key, and StartAfter in V2 responses when url encoding is requested, but StartAfter was echoed raw. The echo now takes urlPathEscape and the incoming start-after param runs through decodeMarkerEcho, so a client resuming from the emitted form lands on the raw key. * Merge master and reconcile ListMultipartUploads marker encoding Resolve the overlap with the merged encoding-type=url pass: keys and markers now go through the single post-pass so nothing is url-escaped twice. Markers are still token-encoded when the response is not url-encoded, and the key-marker decode checks the query-escaped wire form the endpoint actually emits. * s3: echo the decoded start-after under encoding-type=url The escaped echo of the raw input is not a fixpoint: a client resubmitting the echoed StartAfter decoded it to the literal escape text and resumed at the wrong position. * s3: keep url-mode multipart key-marker decodes off the token path Under encoding-type=url the emitted KeyMarker/NextKeyMarker are the url-escaped raw key, never a base64-prefixed token, so the request decode must unescape only. Running the token decode afterward misread a literal "base64:..." key as an encoded token and resumed at the decoded bytes, skipping the keys in between. --------- Co-authored-by: Chris Lu Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/s3api/filer_multipart.go | 27 +- weed/s3api/filer_multipart_list_test.go | 63 ++++ weed/s3api/s3api_object_handlers_list.go | 134 ++++++- .../s3api_object_handlers_list_token_test.go | 328 ++++++++++++++++++ weed/s3api/s3api_object_handlers_multipart.go | 8 +- weed/s3api/s3api_object_versioning.go | 6 +- 6 files changed, 543 insertions(+), 23 deletions(-) create mode 100644 weed/s3api/s3api_object_handlers_list_token_test.go diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go index 9d9f1f8cc8..299c9899cd 100644 --- a/weed/s3api/filer_multipart.go +++ b/weed/s3api/filer_multipart.go @@ -1420,9 +1420,9 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { output.Upload = append(output.Upload, upload) } if p.more { - last := output.Upload[len(output.Upload)-1] + last := candidates[len(candidates)-1] output.IsTruncated = aws.Bool(true) - output.NextKeyMarker, output.NextUploadIdMarker = last.Key, last.UploadId + output.NextKeyMarker, output.NextUploadIdMarker = aws.String(last.key), aws.String(last.entry.Name) } if aws.StringValue(input.EncodingType) == s3.EncodingTypeUrl { // AWS url-encodes these key-named elements when encoding-type=url is @@ -1436,22 +1436,29 @@ func (p *multipartUploadPage) result() *ListMultipartUploadsResult { for _, upload := range output.Upload { upload.Key = encodeMultipartUploadURLField(upload.Key) } + } else { + output.KeyMarker = encodeListContinuationTokenField(output.KeyMarker) + output.NextKeyMarker = encodeListContinuationTokenField(output.NextKeyMarker) } return output } -// encodeMultipartUploadURLField url-encodes key text the way ListObjects does: -// every "/"-separated segment is query-escaped and the separators are kept, so a -// client recovers the exact key with url.QueryUnescape. +// urlQueryEscape url-encodes key text per segment: every "/"-separated part is +// query-escaped and the separators are kept, so a client recovers the exact key +// with url.QueryUnescape. +func urlQueryEscape(key string) string { + segments := strings.Split(key, "/") + for i, segment := range segments { + segments[i] = url.QueryEscape(segment) + } + return strings.Join(segments, "/") +} + func encodeMultipartUploadURLField(value *string) *string { if value == nil { return nil } - segments := strings.Split(*value, "/") - for i, segment := range segments { - segments[i] = url.QueryEscape(segment) - } - return aws.String(strings.Join(segments, "/")) + return aws.String(urlQueryEscape(*value)) } type ListPartsResult struct { diff --git a/weed/s3api/filer_multipart_list_test.go b/weed/s3api/filer_multipart_list_test.go index c00f38acaf..3d2a73789c 100644 --- a/weed/s3api/filer_multipart_list_test.go +++ b/weed/s3api/filer_multipart_list_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math" + "net/url" "slices" "testing" @@ -143,6 +144,68 @@ func TestSelectMultipartUploadsHoldsOnlyThePage(t *testing.T) { } } +// With encoding-type=url the markers and upload keys take the wire form the +// object listings already use: a client that echoes the escaped marker back +// still resumes at the decoded key. +func TestSelectMultipartUploadsUrlEncodesMarkersAndKeys(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "a\x01b", 10), + uploadEntry("2", "a b", 20), + uploadEntry("3", "b", 30), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + + out := selectMultipartUploads(input, entries) + if *out.Upload[0].Key != "a%01b" || *out.NextUploadIdMarker != "1" { + t.Fatalf("page got key %q upload-id-marker %q", *out.Upload[0].Key, *out.NextUploadIdMarker) + } + marker, err := url.QueryUnescape(*out.NextKeyMarker) + if err != nil || decodeListContinuationToken(marker) != "a\x01b" { + t.Fatalf("next key marker %q does not resume at %q", *out.NextKeyMarker, "a\x01b") + } + + input.KeyMarker = aws.String(decodeListContinuationToken(marker)) + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.KeyMarker != "a%01b" || *out.Upload[0].Key != "a+b" || *out.NextKeyMarker != "a+b" { + t.Fatalf("page got marker %q key %q next %q", *out.KeyMarker, *out.Upload[0].Key, *out.NextKeyMarker) + } + + input.KeyMarker = aws.String("a b") + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.IsTruncated || *out.Upload[0].Key != "b" { + t.Fatalf("last page got truncated=%v key %q", *out.IsTruncated, *out.Upload[0].Key) + } +} + +// A raw key that already carries the token prefix must survive an +// encoding-type=url round trip: under url mode the wire form is the url escape +// alone, so the request decode only unescapes — running the token decode on +// "base64:..." would misread the raw key as an encoded token. +func TestSelectMultipartUploadsUrlTokenPrefixedKeyRoundTrip(t *testing.T) { + entries := []*filer_pb.Entry{ + uploadEntry("1", "base64:AQ", 10), + uploadEntry("2", "base64:z", 20), + } + input := listUploadsInput("", "", "", 1) + input.EncodingType = aws.String(s3.EncodingTypeUrl) + + out := selectMultipartUploads(input, entries) + resume := decodeEscapedEcho(*out.NextKeyMarker, true, urlQueryEscape) + if resume != "base64:AQ" { + t.Fatalf("next key marker %q resumes at %q, want %q", *out.NextKeyMarker, resume, "base64:AQ") + } + + input.KeyMarker = aws.String(resume) + input.UploadIdMarker = out.NextUploadIdMarker + out = selectMultipartUploads(input, entries) + if *out.IsTruncated || len(out.Upload) != 1 || *out.Upload[0].Key != "base64%3Az" { + t.Fatalf("second page got truncated=%v uploads %v", *out.IsTruncated, out.Upload) + } +} + func TestSelectMultipartUploadsCapsRequestedPageSize(t *testing.T) { // max-uploads is caller-controlled; the page stays bounded at the server cap page := newMultipartUploadPage(listUploadsInput("", "", "", 10*maxUploadsList)) diff --git a/weed/s3api/s3api_object_handlers_list.go b/weed/s3api/s3api_object_handlers_list.go index 6b049e0e70..76614932ce 100644 --- a/weed/s3api/s3api_object_handlers_list.go +++ b/weed/s3api/s3api_object_handlers_list.go @@ -2,6 +2,7 @@ package s3api import ( "context" + "encoding/base64" "encoding/xml" "errors" "fmt" @@ -11,7 +12,9 @@ import ( "sort" "strconv" "strings" + "unicode/utf8" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -70,8 +73,8 @@ func toListBucketResultV1(in ListBucketResult) listBucketResultV1 { Metadata: in.Metadata, Name: in.Name, Prefix: in.Prefix, - Marker: in.Marker, - NextMarker: in.NextMarker, + Marker: encodeMarkerEcho(in.Marker, in.EncodingType == s3.EncodingTypeUrl), + NextMarker: encodeMarkerEcho(in.NextMarker, in.EncodingType == s3.EncodingTypeUrl), MaxKeys: in.MaxKeys, Delimiter: in.Delimiter, IsTruncated: in.IsTruncated, @@ -104,9 +107,10 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ return } - marker := continuationToken.string + startAfterKey := decodeMarkerEcho(startAfter, encodingTypeUrl) + marker := decodeListContinuationToken(continuationToken.string) if !continuationToken.set { - marker = startAfter + marker = startAfterKey } // Adjust marker if it ends with delimiter to skip all entries with that prefix @@ -145,27 +149,139 @@ func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Requ IsTruncated: response.IsTruncated, KeyCount: len(response.Contents) + len(response.CommonPrefixes), MaxKeys: uint16(response.MaxKeys), - NextContinuationToken: response.NextMarker, + NextContinuationToken: encodeListContinuationToken(response.NextMarker), Prefix: response.Prefix, StartAfter: startAfter, } if encodingTypeUrl { responseV2.EncodingType = s3.EncodingTypeUrl + responseV2.StartAfter = urlPathEscape(startAfterKey) } glog.V(3).Infof("ListObjectsV2Handler response: %+v", responseV2) writeSuccessResponseXML(w, r, responseV2) } +// continuationTokenEncodingPrefix tags a NextContinuationToken whose payload is the +// base64url encoding of the raw key. XML 1.0 cannot carry control characters, and the +// XML encoder replaces them with U+FFFD, so a token holding such a key would resume +// past a mangled name and skip every key sorting between the two. Keys that are valid +// XML text still travel as the bare key, so the token is unchanged for them. +const continuationTokenEncodingPrefix = "base64:" + +func encodeListContinuationToken(key string) string { + if key == "" { + return "" + } + if isXMLText(key) && !strings.HasPrefix(key, continuationTokenEncodingPrefix) { + return key + } + return continuationTokenEncodingPrefix + base64.RawURLEncoding.EncodeToString([]byte(key)) +} + +func encodeListContinuationTokenField(value *string) *string { + if value == nil { + return nil + } + return aws.String(encodeListContinuationToken(*value)) +} + +// encodeMarkerEcho renders a decoded request marker for the response echo: +// the token encoding keeps an XML-unsafe key legible, and urlPathEscape applies +// the encoding-type=url form the keys in the same response already carry. +func encodeMarkerEcho(marker string, encodingTypeUrl bool) string { + marker = encodeListContinuationToken(marker) + if encodingTypeUrl { + marker = urlPathEscape(marker) + } + return marker +} + +// decodeMarkerEcho reverses the wire encoding on a marker the client echoes +// back: under encoding-type=url the emitted echo was urlPathEscape'd. Only a +// value that is the canonical escaped form of its unescape could have come +// from that encoder — "a%2541" round-trips to itself, "a%41" does not — so +// anything else stays the raw key the client sent. +func decodeMarkerEcho(marker string, encodingTypeUrl bool) string { + return decodeEscapedEcho(marker, encodingTypeUrl, urlPathEscape) +} + +func decodeEscapedEcho(marker string, encodingTypeUrl bool, escape func(string) string) string { + if !encodingTypeUrl { + return marker + } + if unescaped, err := url.QueryUnescape(marker); err == nil && escape(unescaped) == marker { + return unescaped + } + return marker +} + +func decodeListContinuationToken(token string) string { + key, ok := strings.CutPrefix(token, continuationTokenEncodingPrefix) + if !ok { + return token + } + decoded, err := base64.RawURLEncoding.DecodeString(key) + if err != nil { + // Not a token this server wrote; resume at the token as the raw key. + return token + } + decodedKey := string(decoded) + // encodeListContinuationToken only writes a token for a key that cannot + // travel bare, so a decoded key that could have stayed bare — valid XML + // text without the prefix — means the input was a legacy raw-key token, + // not something to decode. + if isXMLText(decodedKey) && !strings.HasPrefix(decodedKey, continuationTokenEncodingPrefix) { + return token + } + // A decoded key that could not travel bare makes the token ambiguous + // ("base64:AQ" reads as both the encoded form of "\x01" and a literal + // legacy key). The encoded reading wins: preferring the raw one would + // resume past real entries for every token this server writes, while a + // legacy token aliasing the encoded form only re-lists keys. + return decodedKey +} + +// isXMLText reports whether s survives an XML 1.0 round trip unchanged, i.e. the +// encoder does not replace any of its characters with U+FFFD. +func isXMLText(s string) bool { + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + if r == utf8.RuneError && size == 1 { + return false + } + if !isXMLChar(r) { + return false + } + i += size + } + return true +} + +// isXMLChar follows the XML 1.0 Char production the encoding/xml encoder enforces. +func isXMLChar(r rune) bool { + switch { + case r == 0x9 || r == 0xA || r == 0xD: + return true + case r >= 0x20 && r <= 0xD7FF: + return true + case r >= 0xE000 && r <= 0xFFFD: + return true + case r >= 0x10000 && r <= 0x10FFFF: + return true + } + return false +} + func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html // collect parameters bucket, _ := s3_constants.GetBucketAndObject(r) - originalPrefix, marker, delimiter, encodingTypeUrl, maxKeys, allowUnordered, errCode := getListObjectsV1Args(r.URL.Query()) + originalPrefix, listMarker, delimiter, encodingTypeUrl, maxKeys, allowUnordered, errCode := getListObjectsV1Args(r.URL.Query()) - glog.V(2).Infof("ListObjectsV1Handler bucket=%s prefix=%s marker=%s delimiter=%s maxKeys=%d", bucket, originalPrefix, marker, delimiter, maxKeys) + glog.V(2).Infof("ListObjectsV1Handler bucket=%s prefix=%s marker=%s delimiter=%s maxKeys=%d", bucket, originalPrefix, listMarker, delimiter, maxKeys) if errCode != s3err.ErrNone { s3err.WriteErrorResponse(w, r, errCode) @@ -181,8 +297,8 @@ func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Requ } // Adjust marker if it ends with delimiter to skip all entries with that prefix - requestMarker := marker - marker = adjustMarkerForDelimiter(marker, originalPrefix, delimiter) + requestMarker := decodeListContinuationToken(decodeMarkerEcho(listMarker, encodingTypeUrl)) + marker := adjustMarkerForDelimiter(requestMarker, originalPrefix, delimiter) response, err := s3a.listFilerEntries(r.Context(), listObjectsRequest{ bucket: bucket, diff --git a/weed/s3api/s3api_object_handlers_list_token_test.go b/weed/s3api/s3api_object_handlers_list_token_test.go new file mode 100644 index 0000000000..75ffbdcf9f --- /dev/null +++ b/weed/s3api/s3api_object_handlers_list_token_test.go @@ -0,0 +1,328 @@ +package s3api + +import ( + "encoding/xml" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/gorilla/mux" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type listV2TestPage struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + StartAfter string `xml:"StartAfter"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +func listTokenTestFile(name string) *filer_pb.Entry { + return &filer_pb.Entry{Name: name, Attributes: &filer_pb.FuseAttributes{}} +} + +func newListTokenTestServer(t *testing.T, keys []string) *S3ApiServer { + t.Helper() + entries := make([]*filer_pb.Entry, 0, len(keys)) + for _, key := range keys { + entries = append(entries, listTokenTestFile(key)) + } + filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{ + "/buckets/test": entries, + }} + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.option.BucketsPath = "/buckets" + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test"}) + return s3a +} + +func listV2Page(t *testing.T, s3a *S3ApiServer, query url.Values) listV2TestPage { + t.Helper() + r := httptest.NewRequest(http.MethodGet, "/test?"+query.Encode(), nil) + r = mux.SetURLVars(r, map[string]string{"bucket": "test"}) + w := httptest.NewRecorder() + s3a.ListObjectsV2Handler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var page listV2TestPage + require.NoError(t, xml.Unmarshal(w.Body.Bytes(), &page), w.Body.String()) + return page +} + +// A page that ends on a key XML 1.0 cannot carry must still resume right after +// that key. The token used to be the raw key, which the XML encoder turned into +// U+FFFD, so the next page resumed past the mangled name and skipped every key +// sorting between it and the real one. +func TestListObjectsV2ContinuationTokenWithXMLUnsafeKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + + for _, encodingTypeURL := range []bool{false, true} { + t.Run("encoding-type=url:"+strconv.FormatBool(encodingTypeURL), func(t *testing.T) { + s3a := newListTokenTestServer(t, keys) + + var got []string + token := "" + for page := 0; ; page++ { + require.Less(t, page, len(keys), "listing did not terminate") + query := url.Values{"list-type": {"2"}, "max-keys": {"2"}} + if encodingTypeURL { + query.Set("encoding-type", "url") + } + if token != "" { + query.Set("continuation-token", token) + } + result := listV2Page(t, s3a, query) + for _, content := range result.Contents { + key := content.Key + if encodingTypeURL { + decoded, err := url.QueryUnescape(key) + require.NoError(t, err) + key = decoded + } + got = append(got, key) + } + if !result.IsTruncated { + break + } + require.NotEmpty(t, result.NextContinuationToken) + token = result.NextContinuationToken + } + + if encodingTypeURL { + assert.Equal(t, keys, got) + return + } + // Without encoding-type=url the keys themselves still lose control + // characters to the XML encoder, but no key may be skipped. + require.Len(t, got, len(keys)) + assert.Equal(t, "c", got[len(got)-1]) + }) + } +} + +// The StartAfter echo is the wire encoding of the decoded key, so a client +// that feeds the echo back lands on the same resume point instead of +// drifting to the literal escape text. +func TestListObjectsV2StartAfterEchoStable(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a b", "c"}) + for _, sent := range []string{"a%20b", "a b"} { + result := listV2Page(t, s3a, url.Values{ + "list-type": {"2"}, "encoding-type": {"url"}, "start-after": {sent}, + }) + assert.Equal(t, "a%20b", result.StartAfter, "start-after %q", sent) + require.Len(t, result.Contents, 1) + assert.Equal(t, "c", result.Contents[0].Key) + } +} + +// A token for a key that XML carries unchanged stays the bare key. +func TestListObjectsV2ContinuationTokenUnchangedForNormalKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + s3a := newListTokenTestServer(t, keys) + + result := listV2Page(t, s3a, url.Values{"list-type": {"2"}, "max-keys": {"5"}}) + require.True(t, result.IsTruncated) + assert.Equal(t, "b", result.NextContinuationToken) +} + +func TestListObjectsV2ContinuationTokenRoundTrip(t *testing.T) { + for _, key := range []string{"a\x01b", "a\x1fb", "a\xffb", "base64:literal", "dir/file", "a b", "b"} { + token := encodeListContinuationToken(key) + assert.Equal(t, key, decodeListContinuationToken(token), "token %q", token) + } + for _, key := range []string{"b", "dir/file", "a b"} { + assert.Equal(t, key, encodeListContinuationToken(key)) + } + // Tokens written before the encoding existed are still bare keys. + assert.Equal(t, "dir/file", decodeListContinuationToken("dir/file")) +} + +func TestDecodeListContinuationTokenKeepsForeignTokensRaw(t *testing.T) { + // A prefix-looking token that does not decode is a key, not a token. + assert.Equal(t, "base64:%%%", decodeListContinuationToken("base64:%%%")) + // An empty payload is never something encodeListContinuationToken wrote. + assert.Equal(t, "base64:", decodeListContinuationToken("base64:")) + // A legacy bare token is a raw key even when its suffix is valid base64: + // the decoded key could have traveled bare, so the input never was an + // encoded token. + for _, legacy := range []string{"base64:Yw", "base64:YQ", "base64:cGFy", "base64:a2V5LXdpdGg"} { + assert.Equal(t, legacy, decodeListContinuationToken(legacy), "token %q", legacy) + } +} + +// Some tokens are ambiguous: "base64:AQ" is both a possible legacy bare key +// and the encoded form of "\x01". The encoded reading wins because a decoded +// key that cannot travel bare is a shape only encodeListContinuationToken +// writes — and a wrong raw reading skips entries, while a wrong encoded +// reading only re-lists them. +func TestDecodeListContinuationTokenResolvesCollisionEncoded(t *testing.T) { + assert.Equal(t, "\x01", decodeListContinuationToken("base64:AQ")) + assert.Equal(t, "\xff", decodeListContinuationToken("base64:_w")) + assert.Equal(t, "base64:AQ", decodeListContinuationToken("base64:YmFzZTY0OkFR")) +} + +type listV1TestPage struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + Marker string `xml:"Marker"` + NextMarker string `xml:"NextMarker"` + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` +} + +func listV1Page(t *testing.T, s3a *S3ApiServer, query url.Values) listV1TestPage { + t.Helper() + r := httptest.NewRequest(http.MethodGet, "/test?"+query.Encode(), nil) + r = mux.SetURLVars(r, map[string]string{"bucket": "test"}) + w := httptest.NewRecorder() + s3a.ListObjectsV1Handler(w, r) + require.Equal(t, http.StatusOK, w.Code, w.Body.String()) + var page listV1TestPage + require.NoError(t, xml.Unmarshal(w.Body.Bytes(), &page), w.Body.String()) + return page +} + +// The V1 marker carries the same resume position as the V2 token, so it gets +// the same treatment: a page ending on an XML-unsafe key must still resume +// right after it. +func TestListObjectsV1MarkerWithXMLUnsafeKeys(t *testing.T) { + keys := []string{"a\x01b", "a\x02b", "a\x1fb", "a b", "b", "c"} + s3a := newListTokenTestServer(t, keys) + + var got []string + marker := "" + for page := 0; ; page++ { + require.Less(t, page, len(keys), "listing did not terminate") + query := url.Values{"max-keys": {"2"}, "encoding-type": {"url"}} + if marker != "" { + query.Set("marker", marker) + } + result := listV1Page(t, s3a, query) + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + if !result.IsTruncated { + break + } + require.NotEmpty(t, result.NextMarker) + marker = result.NextMarker + } + require.Equal(t, keys, got) +} + +// A url-encoded marker is only a wire echo when re-escaping its unescaped +// form reproduces it: a real echo decodes, a raw key merely containing '%' +// or a malformed escape passes through untouched. +func TestDecodeMarkerEchoKeepsNonCanonicalRawKey(t *testing.T) { + assert.Equal(t, "a b", decodeMarkerEcho("a%20b", true)) + assert.Equal(t, "a%41", decodeMarkerEcho("a%2541", true)) + assert.Equal(t, "a%41", decodeMarkerEcho("a%41", true)) + assert.Equal(t, "a%2G", decodeMarkerEcho("a%2G", true)) + assert.Equal(t, "a%20b", decodeMarkerEcho("a%20b", false)) +} + +// Under encoding-type=url AWS encodes the StartAfter echo the same as the +// listed keys; a client resuming from the emitted form lands on the raw key. +func TestListObjectsV2UrlStartAfterEchoAndResume(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a b", "a b0", "a c", "b"}) + + result := listV2Page(t, s3a, url.Values{"list-type": {"2"}, "encoding-type": {"url"}, "start-after": {"a b"}}) + assert.Equal(t, urlPathEscape("a b"), result.StartAfter) + + var got []string + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a b0", "a c", "b"}, got) + + // Echoing the emitted form back resumes at the same raw key. + result = listV2Page(t, s3a, url.Values{"list-type": {"2"}, "encoding-type": {"url"}, "start-after": {result.StartAfter}}) + got = nil + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a b0", "a c", "b"}, got) +} + +// A url-encoded marker that is not the canonical urlPathEscape form of any key +// is the client's raw key, not an encoded echo: "a%41" must resume after that +// literal key instead of decoding to "aA" and skipping the keys between them. +func TestListObjectsV1UrlMarkerKeepsNonCanonicalRawKey(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a%41", "a%42", "aA", "b"}) + + result := listV1Page(t, s3a, url.Values{"encoding-type": {"url"}, "marker": {"a%41"}}) + var got []string + for _, content := range result.Contents { + key, err := url.QueryUnescape(content.Key) + require.NoError(t, err) + got = append(got, key) + } + assert.Equal(t, []string{"a%42", "aA", "b"}, got) +} + +// The echoed request marker carries the client's token, not the decoded key: +// echoing the raw key would mangle an XML-unsafe name to U+FFFD, and under +// encoding-type=url the echo takes the same escaped form as the listed keys. +func TestListObjectsV1MarkerEchoStaysXMLSafe(t *testing.T) { + s3a := newListTokenTestServer(t, []string{"a\x01b", "b", "c"}) + + token := encodeListContinuationToken("a\x01b") + result := listV1Page(t, s3a, url.Values{"max-keys": {"2"}, "marker": {token}}) + assert.Equal(t, token, result.Marker) + + result = listV1Page(t, s3a, url.Values{"max-keys": {"2"}, "marker": {"a b"}, "encoding-type": {"url"}}) + assert.Equal(t, urlPathEscape("a b"), result.Marker) +} + +// A raw-key token resuming at an XML-unsafe prefix must not lose keys when a +// sibling directory shares the key's leading bytes: the start window begins at +// the marker's first path segment and descends only into blocks that can still +// hold keys past it. +func TestListObjectsV2ResumeXMLUnsafePrefixWithSiblingDir(t *testing.T) { + filer := &fakeBucketDeleteFiler{entriesByDir: map[string][]*filer_pb.Entry{ + "/buckets/test/x": { + {Name: "a\x01", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, + listTokenTestFile("a\x01b"), + {Name: "a\x01b", IsDirectory: true, Attributes: &filer_pb.FuseAttributes{}}, + listTokenTestFile("a\x01b0"), + }, + "/buckets/test/x/a\x01": { + listTokenTestFile("b"), + }, + "/buckets/test/x/a\x01b": { + listTokenTestFile("0"), + }, + }} + s3a := newFailoverTestServer(t, startFakeFiler(t, filer)) + s3a.option.BucketsPath = "/buckets" + s3a.bucketConfigCache = NewBucketConfigCache(time.Minute) + s3a.bucketConfigCache.Set("test", &BucketConfig{Name: "test"}) + + result := listV2Page(t, s3a, url.Values{ + "list-type": {"2"}, + "encoding-type": {"url"}, + "prefix": {"x/a\x01b"}, + "max-keys": {"1"}, + "continuation-token": {"x/a\x01b"}, + }) + require.Len(t, result.Contents, 1) + got, err := url.QueryUnescape(result.Contents[0].Key) + require.NoError(t, err) + assert.Equal(t, "x/a\x01b/0", got) + assert.True(t, result.IsTruncated) +} diff --git a/weed/s3api/s3api_object_handlers_multipart.go b/weed/s3api/s3api_object_handlers_multipart.go index 945d7862d6..ddb349bdeb 100644 --- a/weed/s3api/s3api_object_handlers_multipart.go +++ b/weed/s3api/s3api_object_handlers_multipart.go @@ -260,11 +260,17 @@ func (s3a *S3ApiServer) ListMultipartUploadsHandler(w http.ResponseWriter, r *ht return } - prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) + prefix, multipartKeyMarker, uploadIDMarker, delimiter, maxUploads, encodingType := getBucketMultipartResources(r.URL.Query()) if maxUploads < 0 { s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxUploads) return } + // The decode mirrors the response encoding: under encoding-type=url the + // markers go out url-escaped, otherwise they carry the token form. + keyMarker := decodeListContinuationToken(multipartKeyMarker) + if encodingType == s3.EncodingTypeUrl { + keyMarker = decodeEscapedEcho(multipartKeyMarker, true, urlQueryEscape) + } if keyMarker != "" { // Marker not common with prefix is not implemented. if !strings.HasPrefix(keyMarker, prefix) { diff --git a/weed/s3api/s3api_object_versioning.go b/weed/s3api/s3api_object_versioning.go index b4a7a4bed9..0e4cc950f3 100644 --- a/weed/s3api/s3api_object_versioning.go +++ b/weed/s3api/s3api_object_versioning.go @@ -448,12 +448,12 @@ func (s3a *S3ApiServer) splitIntoResult(combinedList []versionListItem, bucket, result := &S3ListObjectVersionsResult{ Name: bucket, Prefix: prefix, - KeyMarker: keyMarker, + KeyMarker: encodeListContinuationToken(keyMarker), VersionIdMarker: versionIdMarker, MaxKeys: maxKeys, Delimiter: delimiter, IsTruncated: isTruncated, - NextKeyMarker: nextKeyMarker, + NextKeyMarker: encodeListContinuationToken(nextKeyMarker), NextVersionIdMarker: nextVersionIdMarker, Entries: make([]VersionListEntry, 0, len(combinedList)), } @@ -1906,7 +1906,7 @@ func (s3a *S3ApiServer) ListObjectVersionsHandler(w http.ResponseWriter, r *http prefix := strings.TrimPrefix(originalPrefix, "/") // Note: prefix is used for filtering relative to bucket root, so no leading slash needed - keyMarker := query.Get("key-marker") + keyMarker := decodeListContinuationToken(query.Get("key-marker")) versionIdMarker := query.Get("version-id-marker") delimiter := query.Get("delimiter") From 1d2d9dfed26f4c771c6c16f49b2dd62751e3d990 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 08:13:36 -0700 Subject: [PATCH 68/70] test(fuse): raise mini/filer and mount readiness waits to 90s (#12037) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same startup-timing flake class as the Makefile waits: volume/filer readiness at 30s is marginal on loaded runners — observed failing on a github-hosted runner too (TestPOSIXCompliance + TestExternalPOSIXSuites). The process-death early return still reports real startup failures quickly. --- test/fuse_integration/framework_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/fuse_integration/framework_test.go b/test/fuse_integration/framework_test.go index 3141d9c51d..a7662e96e0 100644 --- a/test/fuse_integration/framework_test.go +++ b/test/fuse_integration/framework_test.go @@ -153,7 +153,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Wait for filer to be ready (mini starts all services on filerPort) - if err := f.waitForService(f.miniProcess, f.filerAddr, 30*time.Second); err != nil { + if err := f.waitForService(f.miniProcess, f.filerAddr, 90*time.Second); err != nil { f.dumpLog("mini") return fmt.Errorf("weed mini not ready: %v", err) } @@ -164,7 +164,7 @@ func (f *FuseTestFramework) Setup(config *TestConfig) error { } // Wait for mount to be ready - if err := f.waitForMount(30 * time.Second); err != nil { + if err := f.waitForMount(90 * time.Second); err != nil { f.dumpLog("mount") return fmt.Errorf("FUSE mount not ready: %v", err) } From 4c2c0ff21caf265395bc362ab6748ca87be8d820 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 24 Oct 2026 08:20:22 -0700 Subject: [PATCH 69/70] s3: fix wildcard matcher skipping '*' targets in policy patterns (#12034) * s3: check pattern '*' before literal equality in matchWildcardString When the target string carried a '*' at the offset of the pattern's '*', the literal-equality case consumed it as a one-character match and no backtrack point was recorded, so the comparison returned false. A Deny Resource carrying a policy variable, or an Allow carrying NotResource, could then be sidestepped by an object key containing '*' at the aligned offset. * test: cover literal asterisks in wildcard match targets --------- Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/util/wildcard/wildcard_matcher.go | 8 +++++--- weed/util/wildcard/wildcard_matcher_test.go | 6 ++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/weed/util/wildcard/wildcard_matcher.go b/weed/util/wildcard/wildcard_matcher.go index 0d74ffe6c4..5b2b85f6da 100644 --- a/weed/util/wildcard/wildcard_matcher.go +++ b/weed/util/wildcard/wildcard_matcher.go @@ -155,13 +155,15 @@ func matchWildcardString(pattern, str string) bool { for targetIndex < len(str) { switch { - case patternIndex < len(pattern) && (pattern[patternIndex] == '?' || pattern[patternIndex] == str[targetIndex]): - targetIndex++ - patternIndex++ + // '*' must be checked before literal equality so a '*' in the target + // cannot consume the pattern wildcard without a backtrack point case patternIndex < len(pattern) && pattern[patternIndex] == '*': lastStarIndex = patternIndex lastStarMatchIndex = targetIndex patternIndex++ + case patternIndex < len(pattern) && (pattern[patternIndex] == '?' || pattern[patternIndex] == str[targetIndex]): + targetIndex++ + patternIndex++ case lastStarIndex != -1: patternIndex = lastStarIndex + 1 lastStarMatchIndex++ diff --git a/weed/util/wildcard/wildcard_matcher_test.go b/weed/util/wildcard/wildcard_matcher_test.go index 26f5f43a89..d318206a38 100644 --- a/weed/util/wildcard/wildcard_matcher_test.go +++ b/weed/util/wildcard/wildcard_matcher_test.go @@ -27,6 +27,12 @@ func TestMatchesWildcard(t *testing.T) { {"Pattern with regex special chars", "test[abc]", "test[abc]", true}, {"Pattern with dots", "test.txt", "test.txt", true}, {"Pattern with dots and wildcard", "*.txt", "test.txt", true}, + {"Star in target at pattern star offset", "a/*", "a/*b", true}, + {"Star in target equals pattern", "a/*", "a/*", true}, + {"Star in target past pattern star offset", "a/*", "a/x*y", true}, + {"Question mark matches literal star", "a/?c", "a/*c", true}, + {"Star in target is not a wildcard", "ab*", "a*b", false}, + {"ARN pattern with star in key", "arn:aws:s3:::bucket/finance/*", "arn:aws:s3:::bucket/finance/*star.txt", true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 81c90cb446480cab38bd93ebf768ce68e92e79b9 Mon Sep 17 00:00:00 2001 From: Konstantin Adzer <47555435+AdzerKI@users.noreply.github.com> Date: Sat, 24 Oct 2026 18:22:29 +0300 Subject: [PATCH 70/70] filer.backup: retry a failed checkpoint read instead of replaying from the epoch (#12036) * filer.backup: retry a failed checkpoint read instead of replaying from the epoch Fixes #12035 * test: reset sink.local.directory in backup offset test cleanup The new test sets sink.local.directory on the shared viper but only reset the enabled flag, leaking a key that points at a deleted t.TempDir() to later tests in the package. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: cover the retried attempt re-reading the checkpoint The fake filer now fails only its first KvGet, and the test runs a second doFilerBackup attempt asserting the checkpoint is re-read and the attempt proceeds past the offset stage. LookupDirectoryEntry is stubbed so the SSE-S3 KEK probe returns a miss instead of retrying 10x2s. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: user.email Co-authored-by: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- weed/command/filer_backup.go | 2 +- weed/command/filer_backup_offset_test.go | 61 +++++++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/weed/command/filer_backup.go b/weed/command/filer_backup.go index 060c5fc837..f184dd23dc 100644 --- a/weed/command/filer_backup.go +++ b/weed/command/filer_backup.go @@ -156,7 +156,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti } else { lastOffsetTsNs, err := getOffsetWithFallback(grpcDialOption, sourceFiler, BackupKeyPrefix, sinkId, BackupKeyPrefix, legacySinkId) if err != nil { - glog.V(0).Infof("starting from %v (offset read failed: %v)", startFrom, err) + return 0, fmt.Errorf("read backup offset: %w", err) } else if lastOffsetTsNs > 0 { startFrom = time.Unix(0, lastOffsetTsNs) glog.V(0).Infof("resuming from %v", startFrom) diff --git a/weed/command/filer_backup_offset_test.go b/weed/command/filer_backup_offset_test.go index 7ae18a3c36..e42efdaa2b 100644 --- a/weed/command/filer_backup_offset_test.go +++ b/weed/command/filer_backup_offset_test.go @@ -3,11 +3,14 @@ package command import ( "context" "net" + "strings" "sync" + "sync/atomic" "testing" "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/util" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -34,14 +37,35 @@ func (s *kvFilerServer) KvPut(_ context.Context, req *filer_pb.KvPutRequest) (*f return &filer_pb.KvPutResponse{}, nil } +type failingKvFilerServer struct { + filer_pb.UnimplementedSeaweedFilerServer + calls atomic.Int32 +} + +func (s *failingKvFilerServer) KvGet(context.Context, *filer_pb.KvGetRequest) (*filer_pb.KvGetResponse, error) { + if s.calls.Add(1) == 1 { + return &filer_pb.KvGetResponse{Error: "kv store unavailable"}, nil + } + return &filer_pb.KvGetResponse{}, nil +} + +// a miss reads as ErrNotFound so the retried attempt skips the KEK retry loop +func (s *failingKvFilerServer) LookupDirectoryEntry(context.Context, *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { + return &filer_pb.LookupDirectoryEntryResponse{}, nil +} + func startKvFiler(t *testing.T) (pb.ServerAddress, grpc.DialOption) { + return startFiler(t, &kvFilerServer{kv: map[string][]byte{}}) +} + +func startFiler(t *testing.T, server filer_pb.SeaweedFilerServer) (pb.ServerAddress, grpc.DialOption) { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatalf("listen: %v", err) } grpcServer := grpc.NewServer() - filer_pb.RegisterSeaweedFilerServer(grpcServer, &kvFilerServer{kv: map[string][]byte{}}) + filer_pb.RegisterSeaweedFilerServer(grpcServer, server) go func() { _ = grpcServer.Serve(listener) }() t.Cleanup(func() { grpcServer.Stop() }) grpcPort := listener.Addr().(*net.TCPAddr).Port @@ -110,3 +134,38 @@ func TestBackupOffset_DistinctDestinationsDoNotShare(t *testing.T) { t.Fatalf("B reads (%d, %v), want (2000, nil)", got, err) } } + +func TestFilerBackup_OffsetReadFailureFailsAttempt(t *testing.T) { + server := &failingKvFilerServer{} + filerAddr, dial := startFiler(t, server) + + config := util.GetViper() + config.Set("sink.local.enabled", true) + config.Set("sink.local.directory", t.TempDir()) + t.Cleanup(func() { + config.Set("sink.local.enabled", false) + config.Set("sink.local.directory", "") + }) + + options := filerBackupOptions + address := string(filerAddr) + options.filer = &address + + resumeTsNs, err := doFilerBackup(dial, &options, nil, nil, nil, 1, 1, 0) + if err == nil || !strings.Contains(err.Error(), "kv store unavailable") { + t.Fatalf("doFilerBackup error = %v, want the offset read failure", err) + } + if resumeTsNs != 0 { + t.Fatalf("resume position = %d, want 0", resumeTsNs) + } + + // the next attempt re-reads the checkpoint and gets past it once the KV + // recovers, so it must fail later, not on the offset read + _, err = doFilerBackup(dial, &options, nil, nil, nil, 1, 2, resumeTsNs) + if err == nil || strings.Contains(err.Error(), "kv store unavailable") { + t.Fatalf("retried doFilerBackup error = %v, want a later-stage failure", err) + } + if got := server.calls.Load(); got < 2 { + t.Fatalf("checkpoint reads across attempts = %d, want a re-read", got) + } +}