mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 03:43:41 +01:00
Compare commits
16
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a5245e060 | ||
|
|
aba3eab8d9 | ||
|
|
3c69805f76 | ||
|
|
1dcf721ce3 | ||
|
|
440c50fc65 | ||
|
|
64f92bfa8a | ||
|
|
b0539a2387 | ||
|
|
59cd87f23c | ||
|
|
db9b3145e2 | ||
|
|
c446ea309b | ||
|
|
504230ce1e | ||
|
|
1caae79f1a | ||
|
|
d33c543f91 | ||
|
|
ffccd8a4ed | ||
|
|
7f25ccf119 | ||
|
|
edae1ac31a |
No files matched your search
@@ -1093,6 +1093,8 @@ impl VolumeServer for VolumeGrpcService {
|
||||
self.state.check_maintenance()?;
|
||||
let req = request.into_inner();
|
||||
let mut results = Vec::new();
|
||||
let mut replica_errs: std::collections::HashMap<u32, Result<(), String>> =
|
||||
std::collections::HashMap::new();
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
@@ -1114,6 +1116,23 @@ impl VolumeServer for VolumeGrpcService {
|
||||
}
|
||||
};
|
||||
|
||||
let vid = file_id.volume_id.0;
|
||||
if !replica_errs.contains_key(&vid) {
|
||||
let checked =
|
||||
crate::server::handlers::check_delete_replicas(&self.state, vid).await;
|
||||
replica_errs.insert(vid, checked);
|
||||
}
|
||||
if let Some(Err(e)) = replica_errs.get(&vid) {
|
||||
results.push(volume_server_pb::DeleteResult {
|
||||
file_id: fid_str.clone(),
|
||||
status: 503,
|
||||
error: e.clone(),
|
||||
size: 0,
|
||||
version: 0,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut n = Needle {
|
||||
id: file_id.key,
|
||||
cookie: file_id.cookie,
|
||||
@@ -12965,6 +12984,99 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Go's TestBatchDelete_WaitsForEveryReplica: a replica the master no
|
||||
/// longer lists would miss the delete, so the delete waits for it.
|
||||
#[tokio::test]
|
||||
async fn test_batch_delete_waits_for_every_replica() {
|
||||
use axum::routing::get;
|
||||
|
||||
let (service, _tmp) = make_local_service_with_volume("", None);
|
||||
{
|
||||
let mut store = service.state.store.write().unwrap();
|
||||
store
|
||||
.add_volume(
|
||||
VolumeId(2),
|
||||
DiskType::HardDrive,
|
||||
&VolumeSpec {
|
||||
replica_placement: Some(
|
||||
crate::storage::super_block::ReplicaPlacement::from_string("001")
|
||||
.unwrap(),
|
||||
),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
let mut n = Needle {
|
||||
id: NeedleId(1),
|
||||
cookie: Cookie(0x3344),
|
||||
data: b"hello".to_vec(),
|
||||
data_size: 5,
|
||||
..Needle::default()
|
||||
};
|
||||
store
|
||||
.write_volume_needle(VolumeId(2), &mut n, false)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
let urls = Arc::new(std::sync::Mutex::new(vec!["127.0.0.1:1"]));
|
||||
let app = axum::Router::new().route(
|
||||
"/dir/lookup",
|
||||
get({
|
||||
let urls = urls.clone();
|
||||
move || {
|
||||
let locations: Vec<_> = urls
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|url| serde_json::json!({ "url": url }))
|
||||
.collect();
|
||||
async move {
|
||||
axum::Json(serde_json::json!({
|
||||
"volumeOrFileId": "2",
|
||||
"locations": locations
|
||||
}))
|
||||
}
|
||||
}
|
||||
}),
|
||||
);
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||
*service.state.current_master_url.write().await = format!("127.0.0.1:{port}");
|
||||
|
||||
let fid = needle::FileId::new(VolumeId(2), NeedleId(1), Cookie(0x3344)).to_string();
|
||||
let delete = || async {
|
||||
let resp = service
|
||||
.batch_delete(Request::new(volume_server_pb::BatchDeleteRequest {
|
||||
file_ids: vec![fid.clone()],
|
||||
skip_cookie_check: true,
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
assert_eq!(resp.results.len(), 1, "{:?}", resp.results);
|
||||
(resp.results[0].status, resp.results[0].error.clone())
|
||||
};
|
||||
|
||||
let (status, error) = delete().await;
|
||||
assert_eq!(status, 503, "{error}");
|
||||
assert!(error.contains("replication copy count"), "{error}");
|
||||
{
|
||||
let store = service.state.store.read().unwrap();
|
||||
let (_, v) = store.find_volume(VolumeId(2)).unwrap();
|
||||
let mut n = Needle {
|
||||
id: NeedleId(1),
|
||||
..Needle::default()
|
||||
};
|
||||
v.read_needle(&mut n)
|
||||
.expect("needle deleted while a replica was offline");
|
||||
}
|
||||
|
||||
urls.lock().unwrap().push("127.0.0.2:8080");
|
||||
let (status, error) = delete().await;
|
||||
assert_eq!(status, 202, "{error}");
|
||||
}
|
||||
|
||||
const GO_EC_NEEDLE_NOT_FOUND: &str =
|
||||
"locate in local ec volume: FindNeedleFromEcx: needle not found";
|
||||
|
||||
|
||||
@@ -695,6 +695,44 @@ async fn batch_delete_file_ids(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mirrors Go's checkDeleteReplicas: refuses a delete the master cannot route
|
||||
/// to every replica, as the HTTP delete path does, so an offline replica does
|
||||
/// not keep the needle once it returns.
|
||||
pub(crate) async fn check_delete_replicas(
|
||||
state: &VolumeServerState,
|
||||
vid: u32,
|
||||
) -> Result<(), String> {
|
||||
let copy_count = {
|
||||
let store = state.store.read().unwrap();
|
||||
match store.find_volume(VolumeId(vid)) {
|
||||
Some((_, v)) => v.super_block().replica_placement.get_copy_count(),
|
||||
None => return Ok(()),
|
||||
}
|
||||
};
|
||||
if copy_count <= 1 {
|
||||
return Ok(());
|
||||
}
|
||||
let locations = lookup_volume(
|
||||
&state.http_client,
|
||||
&state.outgoing_http_scheme,
|
||||
&state.current_master().await,
|
||||
vid,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| format!("replicating lookup failed for {}: {}", vid, e))?;
|
||||
let deletable = locations
|
||||
.iter()
|
||||
.filter(|loc| !loc.read_only || loc.read_only_can_delete)
|
||||
.count();
|
||||
if deletable < copy_count as usize {
|
||||
return Err(format!(
|
||||
"replication locations [{}] is less than volume {} replication copy count [{}]",
|
||||
deletable, vid, copy_count
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `query` with the fields named in `overrides` replaced by their values.
|
||||
fn override_query_fields(query: &str, overrides: &[(&str, String)]) -> Result<String, String> {
|
||||
let mut fields = Vec::new();
|
||||
|
||||
@@ -1365,6 +1365,7 @@ fn build_heartbeat_with_ec_status(
|
||||
}
|
||||
|
||||
let (remote_storage_name, remote_storage_key) = vol.remote_storage_name_key();
|
||||
let (read_only, read_only_can_delete) = vol.reported_read_only();
|
||||
let volume_message = master_pb::VolumeInformationMessage {
|
||||
id: vol.id.0,
|
||||
size: volume_size,
|
||||
@@ -1372,8 +1373,8 @@ fn build_heartbeat_with_ec_status(
|
||||
file_count: vol.file_count() as u64,
|
||||
delete_count: vol.deleted_count() as u64,
|
||||
deleted_byte_count: vol.deleted_size(),
|
||||
read_only: vol.is_read_only(),
|
||||
read_only_can_delete: vol.is_no_write_can_delete(),
|
||||
read_only,
|
||||
read_only_can_delete,
|
||||
replica_placement: vol.super_block().replica_placement.to_byte() as u32,
|
||||
version: vol.super_block().version.0 as u32,
|
||||
ttl: vol.super_block().ttl.to_u32(),
|
||||
@@ -1819,6 +1820,43 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_heartbeat_low_disk_volume_reports_can_delete() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let dir = temp_dir.path().to_str().unwrap();
|
||||
|
||||
let mut store = Store::new(NeedleMapKind::InMemory);
|
||||
store
|
||||
.add_location(
|
||||
dir,
|
||||
dir,
|
||||
3,
|
||||
DiskType::HardDrive,
|
||||
MinFreeSpace::Percent(1.0),
|
||||
vec![],
|
||||
)
|
||||
.unwrap();
|
||||
store
|
||||
.add_volume(VolumeId(7), DiskType::HardDrive, &VolumeSpec::default())
|
||||
.unwrap();
|
||||
store.locations[0]
|
||||
.is_disk_space_low
|
||||
.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let heartbeat = build_heartbeat(&test_config(), &mut store);
|
||||
assert!(heartbeat.volumes[0].read_only);
|
||||
assert!(heartbeat.volumes[0].read_only_can_delete);
|
||||
|
||||
store
|
||||
.find_volume(VolumeId(7))
|
||||
.unwrap()
|
||||
.1
|
||||
.set_no_write_or_delete(true);
|
||||
let heartbeat = build_heartbeat(&test_config(), &mut store);
|
||||
assert!(heartbeat.volumes[0].read_only);
|
||||
assert!(!heartbeat.volumes[0].read_only_can_delete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_heartbeat_marks_empty_store_as_has_no_volumes() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -3590,6 +3590,13 @@ impl Volume {
|
||||
self.no_write_flags().1
|
||||
}
|
||||
|
||||
/// What the master is told: low disk space still lets deletes through,
|
||||
/// only no_write_or_delete refuses them. Mirrors Go's reportedReadOnly.
|
||||
pub fn reported_read_only(&self) -> (bool, bool) {
|
||||
let (read_only, no_write_or_delete, _, _) = self.read_only_reasons();
|
||||
(read_only, read_only && !no_write_or_delete)
|
||||
}
|
||||
|
||||
pub fn last_compact_revision(&self) -> u16 {
|
||||
self.io.read().unwrap().last_compact_revision
|
||||
}
|
||||
|
||||
@@ -21,12 +21,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// Maximum number of retry attempts for failed deletions
|
||||
// Backed-off retry attempts before a failed deletion moves to the dead letter queues
|
||||
MaxRetryAttempts = 10
|
||||
// Initial retry delay (will be doubled with each attempt)
|
||||
InitialRetryDelay = 5 * time.Minute
|
||||
// Maximum retry delay
|
||||
MaxRetryDelay = 6 * time.Hour
|
||||
// Deletions still failing after MaxRetryAttempts move to a dead letter
|
||||
// queue retried after DeadLetterRetryDelay, then to a second one retried
|
||||
// after DeadLetterFinalRetryDelay, and are dropped if that fails too.
|
||||
DeadLetterRetryDelay = 24 * time.Hour
|
||||
DeadLetterFinalRetryDelay = 48 * time.Hour
|
||||
// Interval for checking retry queue for ready items
|
||||
DeletionRetryPollInterval = 1 * time.Minute
|
||||
// Maximum number of items to process per retry iteration
|
||||
@@ -58,6 +63,8 @@ var retryablePatterns = []string{
|
||||
"lookup failed", // Volume server discovery issues
|
||||
"too many requests", // Rate limiting / backpressure
|
||||
"try again", // Explicit retry suggestion
|
||||
"replication copy count", // A replica is offline; deletes wait for it to return
|
||||
"no space left on device", // A low-disk replica could not append the tombstone yet
|
||||
}
|
||||
|
||||
// DeletionRetryItem represents a file deletion that failed and needs to be retried
|
||||
@@ -162,6 +169,31 @@ func calculateBackoff(retryCount int) time.Duration {
|
||||
return delay
|
||||
}
|
||||
|
||||
const deadLetterQueues = 2
|
||||
|
||||
// deadLetterQueue reports which dead letter queue an item with this retry
|
||||
// count is in: 0 while it is still in the regular retry schedule.
|
||||
func deadLetterQueue(retryCount int) int {
|
||||
if retryCount <= MaxRetryAttempts {
|
||||
return 0
|
||||
}
|
||||
return retryCount - MaxRetryAttempts
|
||||
}
|
||||
|
||||
// retryDelay is the wait before the attempt numbered retryCount. Items past
|
||||
// the last dead letter queue are due at once so the next poll drops them.
|
||||
func retryDelay(retryCount int) time.Duration {
|
||||
switch deadLetterQueue(retryCount) {
|
||||
case 0:
|
||||
return calculateBackoff(retryCount)
|
||||
case 1:
|
||||
return DeadLetterRetryDelay
|
||||
case 2:
|
||||
return DeadLetterFinalRetryDelay
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// AddOrUpdate adds a new failed deletion or updates an existing one
|
||||
// Time complexity: O(log N) for insertion/update
|
||||
func (q *DeletionRetryQueue) AddOrUpdate(fileId string, errorMsg string, ledgerEpoch uint64) {
|
||||
@@ -215,8 +247,7 @@ func (q *DeletionRetryQueue) RequeueForRetry(item *DeletionRetryItem, errorMsg s
|
||||
item.RetryCount++
|
||||
item.LastError = errorMsg
|
||||
|
||||
// Calculate next retry time with exponential backoff
|
||||
delay := calculateBackoff(item.RetryCount)
|
||||
delay := retryDelay(item.RetryCount)
|
||||
item.NextRetryAt = time.Now().Add(delay)
|
||||
item.inFlight = false // Clear in-flight flag
|
||||
glog.V(2).Infof("requeued retry for %s: attempt %d, next retry in %v", item.FileId, item.RetryCount, delay)
|
||||
@@ -248,14 +279,13 @@ func (q *DeletionRetryQueue) GetReadyItems(maxItems int) (ready []*DeletionRetry
|
||||
// Remove from heap but keep in itemIndex with inFlight flag
|
||||
heap.Pop(&q.heap)
|
||||
|
||||
if item.RetryCount <= MaxRetryAttempts {
|
||||
if deadLetterQueue(item.RetryCount) <= deadLetterQueues {
|
||||
item.inFlight = true // Mark as being processed
|
||||
ready = append(ready, item)
|
||||
} else {
|
||||
// Max attempts reached, log and discard completely
|
||||
delete(q.itemIndex, item.FileId)
|
||||
expired = append(expired, item)
|
||||
glog.Warningf("max retry attempts (%d) reached for %s, last error: %s", MaxRetryAttempts, item.FileId, item.LastError)
|
||||
glog.Warningf("giving up deleting %s after %d retries, last error: %s", item.FileId, item.RetryCount-1, item.LastError)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -586,6 +616,7 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
|
||||
|
||||
// Process outcomes - iterate over readyItems to ensure all items are accounted for
|
||||
var successCount, notFoundCount, retryCount, permanentErrorCount int
|
||||
var deadLettered, finalDeadLettered int
|
||||
for _, item := range readyItems {
|
||||
outcome := outcomes[item.FileId]
|
||||
|
||||
@@ -605,6 +636,12 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
|
||||
glog.Warningf("no deletion result for retried file %s, re-queuing to avoid loss", item.FileId)
|
||||
}
|
||||
// Keep in the durable ledger: still not confirmed.
|
||||
switch deadLetterQueue(item.RetryCount + 1) {
|
||||
case 1:
|
||||
deadLettered++
|
||||
case 2:
|
||||
finalDeadLettered++
|
||||
}
|
||||
f.DeletionRetryQueue.RequeueForRetry(item, outcome.errorMsg)
|
||||
case deletionOutcomePermanent:
|
||||
permanentErrorCount++
|
||||
@@ -624,6 +661,12 @@ func (f *Filer) processRetryBatch(readyItems []*DeletionRetryItem, lookupFunc fu
|
||||
if retryCount > 0 {
|
||||
glog.V(1).Infof("retry: %d files still failing, will retry again later", retryCount)
|
||||
}
|
||||
if deadLettered > 0 {
|
||||
glog.Warningf("retry: %d files moved to the dead letter queue, next retry in %v", deadLettered, DeadLetterRetryDelay)
|
||||
}
|
||||
if finalDeadLettered > 0 {
|
||||
glog.Warningf("retry: %d files moved to the second dead letter queue, next retry in %v", finalDeadLettered, DeadLetterFinalRetryDelay)
|
||||
}
|
||||
if permanentErrorCount > 0 {
|
||||
glog.Warningf("retry: %d files failed with permanent errors", permanentErrorCount)
|
||||
}
|
||||
|
||||
@@ -102,49 +102,41 @@ func TestDeletionRetryQueue_OverflowProtection(t *testing.T) {
|
||||
|
||||
func TestDeletionRetryQueue_MaxAttemptsReached(t *testing.T) {
|
||||
queue := NewDeletionRetryQueue()
|
||||
|
||||
// Add item
|
||||
queue.AddOrUpdate("file1", "error", 0)
|
||||
|
||||
// Manually set retry count to max
|
||||
makeReady := func() {
|
||||
queue.lock.Lock()
|
||||
defer queue.lock.Unlock()
|
||||
item, exists := queue.itemIndex["file1"]
|
||||
if !exists {
|
||||
t.Fatal("Item not found in queue")
|
||||
}
|
||||
item.NextRetryAt = time.Now().Add(-1 * time.Second)
|
||||
heap.Fix(&queue.heap, item.heapIndex)
|
||||
}
|
||||
|
||||
queue.lock.Lock()
|
||||
item, exists := queue.itemIndex["file1"]
|
||||
if !exists {
|
||||
queue.lock.Unlock()
|
||||
t.Fatal("Item not found in queue")
|
||||
}
|
||||
item.RetryCount = MaxRetryAttempts
|
||||
item.NextRetryAt = time.Now().Add(-1 * time.Second) // Ready now
|
||||
heap.Fix(&queue.heap, item.heapIndex)
|
||||
queue.itemIndex["file1"].RetryCount = MaxRetryAttempts
|
||||
queue.lock.Unlock()
|
||||
makeReady()
|
||||
|
||||
// Try to get ready items - should be returned for the last retry (attempt #10)
|
||||
readyItems, _ := queue.GetReadyItems(10)
|
||||
if len(readyItems) != 1 {
|
||||
t.Fatalf("Expected 1 item for last retry, got %d", len(readyItems))
|
||||
// The last backed-off retry, then one attempt from each dead letter queue.
|
||||
for _, wantDelay := range []time.Duration{DeadLetterRetryDelay, DeadLetterFinalRetryDelay, 0} {
|
||||
readyItems, expired := queue.GetReadyItems(10)
|
||||
if len(readyItems) != 1 || len(expired) != 0 {
|
||||
t.Fatalf("Expected 1 ready item, got %d ready and %d expired", len(readyItems), len(expired))
|
||||
}
|
||||
queue.RequeueForRetry(readyItems[0], "still offline")
|
||||
if delay := time.Until(readyItems[0].NextRetryAt); delay > wantDelay || delay < wantDelay-time.Minute {
|
||||
t.Fatalf("attempt %d: expected next retry in ~%v, got %v", readyItems[0].RetryCount, wantDelay, delay)
|
||||
}
|
||||
makeReady()
|
||||
}
|
||||
|
||||
// Requeue it, which will increment its retry count beyond the max
|
||||
queue.RequeueForRetry(readyItems[0], "final error")
|
||||
|
||||
// Manually make it ready again
|
||||
queue.lock.Lock()
|
||||
item, exists = queue.itemIndex["file1"]
|
||||
if !exists {
|
||||
queue.lock.Unlock()
|
||||
t.Fatal("Item not found in queue after requeue")
|
||||
readyItems, expired := queue.GetReadyItems(10)
|
||||
if len(readyItems) != 0 || len(expired) != 1 {
|
||||
t.Errorf("Expected the item to be dropped after the dead letter queues, got %d ready and %d expired", len(readyItems), len(expired))
|
||||
}
|
||||
item.NextRetryAt = time.Now().Add(-1 * time.Second)
|
||||
heap.Fix(&queue.heap, item.heapIndex)
|
||||
queue.lock.Unlock()
|
||||
|
||||
// Now it should be discarded (retry count is 11, exceeds max of 10)
|
||||
readyItems, _ = queue.GetReadyItems(10)
|
||||
if len(readyItems) != 0 {
|
||||
t.Errorf("Expected 0 items (max attempts exceeded), got %d", len(readyItems))
|
||||
}
|
||||
|
||||
// Should be removed from queue
|
||||
if queue.Size() != 0 {
|
||||
t.Errorf("Expected queue size 0 after max attempts exceeded, got %d", queue.Size())
|
||||
}
|
||||
@@ -192,6 +184,9 @@ func TestIsRetryableError(t *testing.T) {
|
||||
{"service unavailable", true, "service unavailable"},
|
||||
{"i/o timeout", true, "I/O timeout"},
|
||||
{"broken pipe", true, "broken pipe"},
|
||||
{"replication locations [1] is less than volume 9 replication copy count [2]", true, "replica offline"},
|
||||
{"write /data/9.dat: no space left on device", true, "disk full"},
|
||||
{"No space left on device (os error 28)", true, "disk full (rust)"},
|
||||
{"not found", false, "not found (not retryable)"},
|
||||
{"invalid file id", false, "invalid input (not retryable)"},
|
||||
{"", false, "empty error"},
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/topology"
|
||||
)
|
||||
|
||||
func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.BatchDeleteRequest) (*volume_server_pb.BatchDeleteResponse, error) {
|
||||
@@ -24,6 +25,7 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
|
||||
}
|
||||
|
||||
now := uint64(time.Now().Unix())
|
||||
replicaErrs := make(map[needle.VolumeId]error)
|
||||
|
||||
for _, fid := range req.FileIds {
|
||||
vid, id_cookie, err := operation.ParseFileId(fid)
|
||||
@@ -56,6 +58,14 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
|
||||
}
|
||||
}
|
||||
|
||||
if err := vs.checkDeleteReplicas(volumeId, replicaErrs); err != nil {
|
||||
resp.Results = append(resp.Results, &volume_server_pb.DeleteResult{
|
||||
FileId: fid,
|
||||
Status: http.StatusServiceUnavailable,
|
||||
Error: err.Error(),
|
||||
})
|
||||
continue
|
||||
}
|
||||
ecVolume, isEcVolume := vs.store.FindEcVolume(volumeId)
|
||||
if !req.SkipCookieCheck {
|
||||
cookie := n.Cookie
|
||||
@@ -145,3 +155,18 @@ func (vs *VolumeServer) BatchDelete(ctx context.Context, req *volume_server_pb.B
|
||||
return resp, nil
|
||||
|
||||
}
|
||||
|
||||
// checkDeleteReplicas refuses a delete the master cannot route to every
|
||||
// replica, as the HTTP delete path does, so an offline replica does not keep
|
||||
// the needle once it returns.
|
||||
func (vs *VolumeServer) checkDeleteReplicas(volumeId needle.VolumeId, checked map[needle.VolumeId]error) error {
|
||||
if err, found := checked[volumeId]; found {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
if v := vs.store.GetVolume(volumeId); v != nil && v.ReplicaPlacement.GetCopyCount() > 1 {
|
||||
_, err = topology.GetRemoteReplications(vs.store, vs.grpcDialOption, volumeId, vs.GetMaster)
|
||||
}
|
||||
checked[volumeId] = err
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,109 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/peer"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
)
|
||||
|
||||
type replicaLookupMaster struct {
|
||||
master_pb.UnimplementedSeaweedServer
|
||||
mu sync.Mutex
|
||||
locations []*master_pb.Location
|
||||
}
|
||||
|
||||
func (m *replicaLookupMaster) LookupVolume(_ context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
resp := &master_pb.LookupVolumeResponse{}
|
||||
for _, vid := range req.VolumeOrFileIds {
|
||||
resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
|
||||
VolumeOrFileId: vid,
|
||||
Locations: m.locations,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (m *replicaLookupMaster) setLocations(locations ...*master_pb.Location) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.locations = locations
|
||||
}
|
||||
|
||||
// A replica the master no longer lists would miss the delete and keep the
|
||||
// needle when it returns, so the delete waits until every replica is back.
|
||||
func TestBatchDelete_WaitsForEveryReplica(t *testing.T) {
|
||||
const vid = needle.VolumeId(9)
|
||||
const cookie = uint32(0x12345678)
|
||||
|
||||
store := newVolumeCopyTestStore(t, t.TempDir())
|
||||
if err := store.AddVolume(vid, "", storage.NeedleMapInMemory, "001", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
|
||||
t.Fatalf("add volume: %v", err)
|
||||
}
|
||||
data := []byte("hello")
|
||||
if _, err := store.WriteVolumeNeedle(vid, &needle.Needle{Id: 1, Cookie: types.Cookie(cookie), Data: data, Checksum: needle.NewCRC(data)}, true, false); err != nil {
|
||||
t.Fatalf("write needle: %v", err)
|
||||
}
|
||||
|
||||
self := &master_pb.Location{Url: "127.0.0.1:0"}
|
||||
peerReplica := &master_pb.Location{Url: "127.0.0.2:8080"}
|
||||
master := &replicaLookupMaster{}
|
||||
master.setLocations(self)
|
||||
masterGrpcPort := serveGrpc(t, func(s *grpc.Server) {
|
||||
master_pb.RegisterSeaweedServer(s, master)
|
||||
})
|
||||
|
||||
vs := &VolumeServer{
|
||||
store: store,
|
||||
guard: security.NewGuard(nil, "", 0, "", 0),
|
||||
grpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
}
|
||||
vs.setCurrentMaster(pb.NewServerAddress("127.0.0.1", masterGrpcPort-10000, masterGrpcPort))
|
||||
|
||||
ctx := peer.NewContext(context.Background(), &peer.Peer{
|
||||
Addr: &net.TCPAddr{IP: net.ParseIP("127.0.0.1"), Port: 12345},
|
||||
})
|
||||
req := &volume_server_pb.BatchDeleteRequest{
|
||||
FileIds: []string{needle.NewFileId(vid, 1, cookie).String()},
|
||||
SkipCookieCheck: true,
|
||||
}
|
||||
|
||||
resp, err := vs.BatchDelete(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("BatchDelete: %v", err)
|
||||
}
|
||||
if len(resp.Results) != 1 {
|
||||
t.Fatalf("got %d results, want 1", len(resp.Results))
|
||||
}
|
||||
if got := resp.Results[0]; got.Status != http.StatusServiceUnavailable || !strings.Contains(got.Error, "replication copy count") {
|
||||
t.Fatalf("with a replica offline: status %d error %q, want %d and a replica count error", got.Status, got.Error, http.StatusServiceUnavailable)
|
||||
}
|
||||
if _, err := store.ReadVolumeNeedle(vid, &needle.Needle{Id: 1}, nil, nil); err != nil {
|
||||
t.Fatalf("needle deleted while a replica was offline: %v", err)
|
||||
}
|
||||
|
||||
master.setLocations(self, peerReplica)
|
||||
resp, err = vs.BatchDelete(ctx, req)
|
||||
if err != nil {
|
||||
t.Fatalf("BatchDelete: %v", err)
|
||||
}
|
||||
if got := resp.Results[0]; got.Status != http.StatusAccepted || got.Error != "" {
|
||||
t.Fatalf("with every replica back: status %d error %q, want %d", got.Status, got.Error, http.StatusAccepted)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"flag"
|
||||
"io"
|
||||
"log"
|
||||
"maps"
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
@@ -31,10 +33,12 @@ func (c *commandVolumeDeleteEmpty) Help() string {
|
||||
volume.deleteEmpty -quietFor=24h -apply
|
||||
volume.deleteEmpty -collectionPattern=important* -quietFor=24h -apply
|
||||
|
||||
This command deletes all empty volumes from one volume server.
|
||||
This command deletes all empty volumes from all volume servers.
|
||||
A volume with no live needles left is empty too, even when its
|
||||
.dat file is still large: compacting it first would only rewrite
|
||||
bytes that are all deleted already.
|
||||
A replicated volume is deleted only when all of its copies are
|
||||
listed and every copy is empty.
|
||||
|
||||
`
|
||||
}
|
||||
@@ -76,31 +80,55 @@ func (c *commandVolumeDeleteEmpty) Do(args []string, commandEnv *CommandEnv, wri
|
||||
quietSeconds := int64(*quietPeriod / time.Second)
|
||||
nowUnixSeconds := time.Now().Unix()
|
||||
|
||||
eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
|
||||
for _, diskInfo := range dn.DiskInfos {
|
||||
for _, v := range diskInfo.VolumeInfos {
|
||||
if isEmptyVolumeDeleteCandidate(v, quietSeconds, nowUnixSeconds, collectionMatcher) {
|
||||
if *applyBalancing {
|
||||
log.Printf("deleting empty volume %d from %s", v.Id, dn.Id)
|
||||
onlyGarbage := v.FileCount > 0 && v.FileCount <= v.DeleteCount
|
||||
// onlyEmpty stays set so a pre-upgrade server checks
|
||||
// emptiness and refuses instead of deleting unseen.
|
||||
if deleteErr := deleteVolume(context.Background(), commandEnv.option.GrpcDialOption, needle.VolumeId(v.Id),
|
||||
pb.NewServerAddressFromDataNode(dn), true, onlyGarbage, false); deleteErr != nil {
|
||||
err = deleteErr
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
log.Printf("empty volume %d from %s", v.Id, dn.Id)
|
||||
}
|
||||
}
|
||||
volumeReplicas, _ := collectVolumeReplicaLocations(topologyInfo)
|
||||
for _, vid := range slices.Sorted(maps.Keys(volumeReplicas)) {
|
||||
replicas := volumeReplicas[vid]
|
||||
if !isEmptyVolumeCopiesDeleteCandidate(replicas, quietSeconds, nowUnixSeconds, collectionMatcher) {
|
||||
continue
|
||||
}
|
||||
var kept []pb.ServerAddress
|
||||
for _, r := range replicas {
|
||||
dn := r.location.dataNode
|
||||
if !*applyBalancing {
|
||||
log.Printf("empty volume %d from %s", vid, dn.Id)
|
||||
continue
|
||||
}
|
||||
log.Printf("deleting empty volume %d from %s", vid, dn.Id)
|
||||
onlyGarbage := r.info.FileCount > 0 && r.info.FileCount <= r.info.DeleteCount
|
||||
// onlyEmpty stays set so a pre-upgrade server checks
|
||||
// emptiness and refuses instead of deleting unseen.
|
||||
server := pb.NewServerAddressFromDataNode(dn)
|
||||
if deleteErr := deleteVolume(context.Background(), commandEnv.option.GrpcDialOption, needle.VolumeId(vid),
|
||||
server, true, onlyGarbage, false); deleteErr != nil {
|
||||
err = deleteErr
|
||||
kept = append(kept, server)
|
||||
}
|
||||
}
|
||||
})
|
||||
// a rerun skips the survivors as under-replicated
|
||||
if len(kept) < len(replicas) {
|
||||
for _, server := range kept {
|
||||
log.Printf("volume %d stays on %s after its other copies were deleted; once it is empty, remove it with volume.delete -node %s -volumeId %d", vid, server, string(server), vid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// isEmptyVolumeCopiesDeleteCandidate requires every copy to be listed and
|
||||
// empty: deleting only the empty ones would leave a copy that holds data, or
|
||||
// one that is offline, as the volume's only copy.
|
||||
func isEmptyVolumeCopiesDeleteCandidate(replicas []*VolumeReplica, quietSeconds, nowUnixSeconds int64, collectionMatcher *wildcard.CollectionMatcher) bool {
|
||||
for _, r := range replicas {
|
||||
rp, err := super_block.NewReplicaPlacementFromByte(byte(r.info.ReplicaPlacement))
|
||||
if err != nil || len(replicas) < rp.GetCopyCount() ||
|
||||
!isEmptyVolumeDeleteCandidate(r.info, quietSeconds, nowUnixSeconds, collectionMatcher) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return len(replicas) > 0
|
||||
}
|
||||
|
||||
func isEmptyVolumeDeleteCandidate(v *master_pb.VolumeInformationMessage, quietSeconds, nowUnixSeconds int64, collectionMatcher *wildcard.CollectionMatcher) bool {
|
||||
return collectionMatcher.Matches(v.Collection) &&
|
||||
// A remote-backed replica shares its cloud object with the others, so
|
||||
|
||||
@@ -39,3 +39,46 @@ func TestIsEmptyVolumeDeleteCandidate(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsEmptyVolumeCopiesDeleteCandidate(t *testing.T) {
|
||||
matcher, err := wildcard.CompileCollectionMatcher("")
|
||||
assert.NoError(t, err)
|
||||
|
||||
const quietSeconds = int64(60)
|
||||
const now = int64(1_000_000)
|
||||
old := now - quietSeconds - 1
|
||||
copyOf := func(rp uint32, v *master_pb.VolumeInformationMessage) *VolumeReplica {
|
||||
v.ReplicaPlacement = rp
|
||||
return &VolumeReplica{location: &location{dataNode: &master_pb.DataNodeInfo{Id: "dn"}}, info: v}
|
||||
}
|
||||
empty := func() *master_pb.VolumeInformationMessage {
|
||||
return &master_pb.VolumeInformationMessage{Size: super_block.SuperBlockSize, ModifiedAtSecond: old}
|
||||
}
|
||||
lowDisk := func() *master_pb.VolumeInformationMessage {
|
||||
return &master_pb.VolumeInformationMessage{Size: super_block.SuperBlockSize, ReadOnly: true, ReadOnlyCanDelete: true, ModifiedAtSecond: old}
|
||||
}
|
||||
withData := func() *master_pb.VolumeInformationMessage {
|
||||
return &master_pb.VolumeInformationMessage{Size: 1 << 20, FileCount: 3, ModifiedAtSecond: old}
|
||||
}
|
||||
protected := func() *master_pb.VolumeInformationMessage {
|
||||
return &master_pb.VolumeInformationMessage{Size: super_block.SuperBlockSize, ReadOnly: true, ModifiedAtSecond: old}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
replicas []*VolumeReplica
|
||||
want bool
|
||||
}{
|
||||
{"single copy volume", []*VolumeReplica{copyOf(0, empty())}, true},
|
||||
{"every copy empty", []*VolumeReplica{copyOf(1, empty()), copyOf(1, lowDisk())}, true},
|
||||
{"other copy holds data", []*VolumeReplica{copyOf(1, lowDisk()), copyOf(1, withData())}, false},
|
||||
{"other copy is missing", []*VolumeReplica{copyOf(1, lowDisk())}, false},
|
||||
{"other copy is protected read-only", []*VolumeReplica{copyOf(1, empty()), copyOf(1, protected())}, false},
|
||||
{"no copies", nil, false},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
assert.Equal(t, tc.want, isEmptyVolumeCopiesDeleteCandidate(tc.replicas, quietSeconds, now, matcher))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -291,7 +291,7 @@ func (s *Store) DeleteCollection(collection string) (e error) {
|
||||
// volume grown and destroyed between two of them was never reported at
|
||||
// all, so nothing else would ever tell the master its slot came free.
|
||||
for _, v := range deleted {
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
s.DeletedVolumesChan <- &master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(v.Id),
|
||||
Collection: v.Collection,
|
||||
@@ -428,7 +428,7 @@ func (s *Store) addVolume(vid needle.VolumeId, collection string, needleMapKind
|
||||
volume.diskId = diskId // Set the disk ID
|
||||
location.SetVolume(vid, volume)
|
||||
glog.V(0).Infof("add volume %d on disk ID %d", vid, diskId)
|
||||
readOnly, _, readOnlyCanDelete, _ := volume.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := volume.reportedReadOnly()
|
||||
s.NewVolumesChan <- &master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(vid),
|
||||
Collection: collection,
|
||||
@@ -523,7 +523,7 @@ func collectStatsForOneLocation(location *DiskLocation) (stats []*VolumeInfo) {
|
||||
|
||||
func collectStatForOneVolume(vid needle.VolumeId, v *Volume) (s *VolumeInfo) {
|
||||
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
s = &VolumeInfo{
|
||||
Id: vid,
|
||||
Collection: v.Collection,
|
||||
@@ -1151,7 +1151,7 @@ func (s *Store) mountVolume(i needle.VolumeId, collection *string, ownLoad bool,
|
||||
return err
|
||||
}
|
||||
}
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
s.NewVolumesChan <- &master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(v.Id),
|
||||
Collection: v.Collection,
|
||||
@@ -1183,7 +1183,7 @@ func (s *Store) UnmountVolume(i needle.VolumeId) error {
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
message := master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(v.Id),
|
||||
Collection: v.Collection,
|
||||
@@ -1289,7 +1289,7 @@ func (s *Store) DeleteVolume(i needle.VolumeId, onlyEmpty bool, onlyGarbage bool
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
message := master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(v.Id),
|
||||
Collection: v.Collection,
|
||||
@@ -1369,7 +1369,7 @@ func (s *Store) deleteVolumeGuarded(i needle.VolumeId, onlyEmpty bool, onlyGarba
|
||||
var deletedMessages []*master_pb.VolumeShortInformationMessage
|
||||
for _, location := range lockedLocations {
|
||||
v := location.volumes[i]
|
||||
readOnly, _, readOnlyCanDelete, _ := v.ReadOnlyReasons()
|
||||
readOnly, readOnlyCanDelete := v.reportedReadOnly()
|
||||
message := &master_pb.VolumeShortInformationMessage{
|
||||
Id: uint32(v.Id),
|
||||
Collection: v.Collection,
|
||||
|
||||
@@ -153,6 +153,30 @@ func TestMountVolumeAnnouncesReadOnlyState(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLowDiskVolumeReportsCanDelete(t *testing.T) {
|
||||
store := newSingleDirStore(t, t.TempDir())
|
||||
defer store.Close()
|
||||
const vid = needle.VolumeId(14)
|
||||
require.NoError(t, store.AddVolume(vid, "", NeedleMapInMemory, "000", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0))
|
||||
v := store.GetVolume(vid)
|
||||
n := newRandomNeedle(1)
|
||||
_, err := store.WriteVolumeNeedle(vid, n, true, false)
|
||||
require.NoError(t, err)
|
||||
|
||||
store.Locations[0].isDiskSpaceLow.Store(true)
|
||||
_, message := v.ToVolumeInformationMessage(nil)
|
||||
require.True(t, message.ReadOnly)
|
||||
require.True(t, message.ReadOnlyCanDelete)
|
||||
require.Equal(t, message.ReadOnlyCanDelete, store.CollectHeartbeat().Volumes[0].ReadOnlyCanDelete)
|
||||
_, err = store.DeleteVolumeNeedle(vid, n)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, store.MarkVolumeReadonly(vid, false, false))
|
||||
_, message = v.ToVolumeInformationMessage(nil)
|
||||
require.True(t, message.ReadOnly)
|
||||
require.False(t, message.ReadOnlyCanDelete)
|
||||
}
|
||||
|
||||
// After a restart a canDelete volume is served from .sdx; its deletes must
|
||||
// still move the counters volume.deleteEmpty and vacuum read, and land them
|
||||
// where the next reload would.
|
||||
|
||||
@@ -549,7 +549,7 @@ func (v *Volume) ToVolumeInformationMessage(into *master_pb.VolumeInformationMes
|
||||
volumeInfo.FileCount = fileCount
|
||||
volumeInfo.DeleteCount = deletedCount
|
||||
volumeInfo.DeletedByteCount = deletedSize
|
||||
volumeInfo.ReadOnly, _, volumeInfo.ReadOnlyCanDelete, _ = v.ReadOnlyReasons()
|
||||
volumeInfo.ReadOnly, volumeInfo.ReadOnlyCanDelete = v.reportedReadOnly()
|
||||
volumeInfo.ReplicaPlacement = uint32(v.ReplicaPlacement.Byte())
|
||||
volumeInfo.Version = uint32(v.Version())
|
||||
volumeInfo.Ttl = v.Ttl.ToUint32()
|
||||
@@ -592,6 +592,13 @@ func (v *Volume) ReadOnlyReasons() (readOnly, noWriteOrDelete, noWriteCanDelete,
|
||||
return noWriteOrDelete || noWriteCanDelete || diskSpaceLow, noWriteOrDelete, noWriteCanDelete, diskSpaceLow
|
||||
}
|
||||
|
||||
// reportedReadOnly is what the master is told: low disk space still lets
|
||||
// deletes through, only noWriteOrDelete refuses them.
|
||||
func (v *Volume) reportedReadOnly() (readOnly, readOnlyCanDelete bool) {
|
||||
readOnly, noWriteOrDelete, _, _ := v.ReadOnlyReasons()
|
||||
return readOnly, readOnly && !noWriteOrDelete
|
||||
}
|
||||
|
||||
var errVolumeUnavailable = errors.New("volume unavailable")
|
||||
|
||||
func (v *Volume) UnavailableError() error {
|
||||
|
||||
@@ -478,21 +478,35 @@ func (t *Topology) deleteEmptyVolumes(grpcDialOption grpc.DialOption, vl *Volume
|
||||
quietSeconds := int64(quietPeriod / time.Second)
|
||||
nowUnixSeconds := time.Now().Unix()
|
||||
for vid, locationList := range todoVolumeMap {
|
||||
eligible := true
|
||||
// A missing replica may still hold data; deleting the listed copies
|
||||
// would leave it as the only one when it returns. A vid still marked
|
||||
// deleting lost copies to an earlier partial delete, so finish it.
|
||||
resuming := vl.IsMarkedDeleting(vid)
|
||||
if len(locationList.list) < vl.rp.GetCopyCount() && !resuming {
|
||||
continue
|
||||
}
|
||||
eligible, changed := true, false
|
||||
for _, dn := range locationList.list {
|
||||
v, err := dn.GetVolumesById(vid)
|
||||
if err == nil && !isQuietEmptyVolume(v, quietSeconds, nowUnixSeconds) {
|
||||
eligible, changed = false, true
|
||||
break
|
||||
}
|
||||
if err != nil || !isEmptyVolumeDeleteCandidate(v, quietSeconds, nowUnixSeconds) {
|
||||
eligible = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if !eligible {
|
||||
// a copy that is only read-only or missing postpones the cleanup
|
||||
if resuming && changed {
|
||||
vl.UnmarkDeleting(vid)
|
||||
}
|
||||
continue
|
||||
}
|
||||
// keep the volume out of assignments for the whole delete: a heartbeat
|
||||
// landing mid-delete must not re-add it while replicas are dropping
|
||||
vl.MarkDeleting(vid)
|
||||
remaining := 0
|
||||
deleted := 0
|
||||
kept := locationList.list[:0]
|
||||
for _, dn := range locationList.list {
|
||||
v, err := dn.GetVolumesById(vid)
|
||||
@@ -501,16 +515,19 @@ func (t *Topology) deleteEmptyVolumes(grpcDialOption grpc.DialOption, vl *Volume
|
||||
glog.V(0).Infof("deleting empty volume %d on %s", vid, dn.ServerAddress())
|
||||
if err = t.deleteEmptyVolume(grpcDialOption, dn, vid, onlyGarbage); err == nil {
|
||||
t.UnRegisterVolumeLayout(v, dn)
|
||||
deleted++
|
||||
continue
|
||||
}
|
||||
}
|
||||
glog.Warningf("delete empty volume %d on %s: %v", vid, dn.ServerAddress(), err)
|
||||
kept = append(kept, dn)
|
||||
remaining++
|
||||
}
|
||||
vl.UnmarkDeleting(vid)
|
||||
// a partly deleted vid stays marked so the next sweep finishes it
|
||||
if len(kept) == 0 || deleted == 0 && !resuming {
|
||||
vl.UnmarkDeleting(vid)
|
||||
}
|
||||
locationList.list = kept
|
||||
if remaining == 0 {
|
||||
if len(kept) == 0 {
|
||||
delete(todoVolumeMap, vid)
|
||||
}
|
||||
}
|
||||
@@ -532,8 +549,11 @@ func (t *Topology) deleteEmptyVolume(grpcDialOption grpc.DialOption, dn *DataNod
|
||||
}
|
||||
|
||||
func isEmptyVolumeDeleteCandidate(v storage.VolumeInfo, quietSeconds, nowUnixSeconds int64) bool {
|
||||
return (!v.ReadOnly || v.ReadOnlyCanDelete) && isQuietEmptyVolume(v, quietSeconds, nowUnixSeconds)
|
||||
}
|
||||
|
||||
func isQuietEmptyVolume(v storage.VolumeInfo, quietSeconds, nowUnixSeconds int64) bool {
|
||||
return v.RemoteStorageName == "" &&
|
||||
(!v.ReadOnly || v.ReadOnlyCanDelete) &&
|
||||
(v.Size <= super_block.SuperBlockSize || v.FileCount > 0 && v.FileCount <= v.DeleteCount) &&
|
||||
v.ModifiedAtSecond > 0 &&
|
||||
v.ModifiedAtSecond+quietSeconds < nowUnixSeconds
|
||||
|
||||
@@ -54,13 +54,18 @@ func TestIsEmptyVolumeDeleteCandidate(t *testing.T) {
|
||||
|
||||
type fakeVolumeDeleteServer struct {
|
||||
volume_server_pb.UnimplementedVolumeServerServer
|
||||
mu sync.Mutex
|
||||
deletes []*volume_server_pb.VolumeDeleteRequest
|
||||
mu sync.Mutex
|
||||
deletes []*volume_server_pb.VolumeDeleteRequest
|
||||
failures int
|
||||
}
|
||||
|
||||
func (f *fakeVolumeDeleteServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if f.failures > 0 {
|
||||
f.failures--
|
||||
return nil, errors.New("volume server busy")
|
||||
}
|
||||
f.deletes = append(f.deletes, req)
|
||||
return &volume_server_pb.VolumeDeleteResponse{}, nil
|
||||
}
|
||||
@@ -211,6 +216,175 @@ func TestDeleteEmptyVolumesSkipsVidWithLiveReplica(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteEmptyVolumesSkipsUnderReplicatedVid(t *testing.T) {
|
||||
fake := &fakeVolumeDeleteServer{}
|
||||
grpcPort, dialOption := startFakeVolumeServer(t, fake)
|
||||
|
||||
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
||||
dn := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1").
|
||||
GetOrCreateDataNode("127.0.0.1", 8080, grpcPort, "127.0.0.1", "dn-online", map[string]uint32{"": 10})
|
||||
rp, err := super_block.NewReplicaPlacementFromString("001")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := storage.VolumeInfo{
|
||||
Id: needle.VolumeId(1),
|
||||
Collection: "c",
|
||||
ModifiedAtSecond: time.Now().Unix() - 7200,
|
||||
Version: needle.GetCurrentVersion(),
|
||||
ReplicaPlacement: rp,
|
||||
Ttl: needle.EMPTY_TTL,
|
||||
ReadOnly: true,
|
||||
ReadOnlyCanDelete: true,
|
||||
}
|
||||
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
topo.RegisterVolumeLayout(v, dn)
|
||||
|
||||
ll := NewVolumeLocationList()
|
||||
ll.list = append(ll.list, dn)
|
||||
todo := map[needle.VolumeId]*VolumeLocationList{v.Id: ll}
|
||||
|
||||
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.ToDiskType(""))
|
||||
topo.deleteEmptyVolumes(dialOption, vl, todo, time.Hour)
|
||||
|
||||
if _, ok := todo[v.Id]; !ok {
|
||||
t.Fatal("vid left the sweep map while a replica is missing")
|
||||
}
|
||||
fake.mu.Lock()
|
||||
defer fake.mu.Unlock()
|
||||
if len(fake.deletes) != 0 {
|
||||
t.Fatalf("VolumeDelete calls = %d, want 0 (the missing replica may still hold data)", len(fake.deletes))
|
||||
}
|
||||
}
|
||||
|
||||
// A delete that removed one copy and failed on the other leaves the vid with
|
||||
// fewer copies than its replica placement; the next sweep must still finish
|
||||
// it instead of treating it as a volume with a replica offline.
|
||||
func TestDeleteEmptyVolumesFinishesPartialDelete(t *testing.T) {
|
||||
okFake := &fakeVolumeDeleteServer{}
|
||||
okPort, dialOption := startFakeVolumeServer(t, okFake)
|
||||
flakyFake := &fakeVolumeDeleteServer{failures: 1}
|
||||
flakyPort, _ := startFakeVolumeServer(t, flakyFake)
|
||||
|
||||
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
||||
rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
|
||||
okDn := rack.GetOrCreateDataNode("127.0.0.1", 8080, okPort, "127.0.0.1", "dn-ok", map[string]uint32{"": 10})
|
||||
flakyDn := rack.GetOrCreateDataNode("127.0.0.1", 8081, flakyPort, "127.0.0.1", "dn-flaky", map[string]uint32{"": 10})
|
||||
rp, err := super_block.NewReplicaPlacementFromString("001")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := storage.VolumeInfo{
|
||||
Id: needle.VolumeId(1),
|
||||
Collection: "c",
|
||||
ModifiedAtSecond: time.Now().Unix() - 7200,
|
||||
Version: needle.GetCurrentVersion(),
|
||||
ReplicaPlacement: rp,
|
||||
Ttl: needle.EMPTY_TTL,
|
||||
}
|
||||
for _, dn := range []*DataNode{okDn, flakyDn} {
|
||||
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
topo.RegisterVolumeLayout(v, dn)
|
||||
}
|
||||
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.ToDiskType(""))
|
||||
|
||||
sweep := func() map[needle.VolumeId]*VolumeLocationList {
|
||||
vl.accessLock.RLock()
|
||||
todo := map[needle.VolumeId]*VolumeLocationList{}
|
||||
for vid, ll := range vl.vid2location {
|
||||
todo[vid] = ll.Copy()
|
||||
}
|
||||
vl.accessLock.RUnlock()
|
||||
topo.deleteEmptyVolumes(dialOption, vl, todo, time.Hour)
|
||||
return todo
|
||||
}
|
||||
|
||||
if todo := sweep(); todo[v.Id] == nil {
|
||||
t.Fatal("vid left the sweep map while a copy delete failed")
|
||||
}
|
||||
if !vl.IsMarkedDeleting(v.Id) {
|
||||
t.Fatal("partly deleted vid lost its deleting mark")
|
||||
}
|
||||
if todo := sweep(); todo[v.Id] != nil {
|
||||
t.Fatal("next sweep left the partly deleted vid behind")
|
||||
}
|
||||
if vl.IsMarkedDeleting(v.Id) {
|
||||
t.Error("fully deleted vid kept its deleting mark")
|
||||
}
|
||||
flakyFake.mu.Lock()
|
||||
defer flakyFake.mu.Unlock()
|
||||
if len(flakyFake.deletes) != 1 {
|
||||
t.Fatalf("VolumeDelete on the flaky copy = %d, want 1 after the retry", len(flakyFake.deletes))
|
||||
}
|
||||
}
|
||||
|
||||
// A survivor that turns read-only between sweeps only postpones the cleanup;
|
||||
// once it is deletable again the next sweep must still finish the vid.
|
||||
func TestDeleteEmptyVolumesResumesAfterReadOnlySurvivor(t *testing.T) {
|
||||
okFake := &fakeVolumeDeleteServer{}
|
||||
okPort, dialOption := startFakeVolumeServer(t, okFake)
|
||||
flakyFake := &fakeVolumeDeleteServer{failures: 1}
|
||||
flakyPort, _ := startFakeVolumeServer(t, flakyFake)
|
||||
|
||||
topo := NewTopology("weedfs", sequence.NewMemorySequencer(), 32*1024, 5, false)
|
||||
rack := topo.GetOrCreateDataCenter("dc1").GetOrCreateRack("rack1")
|
||||
okDn := rack.GetOrCreateDataNode("127.0.0.1", 8080, okPort, "127.0.0.1", "dn-ok", map[string]uint32{"": 10})
|
||||
flakyDn := rack.GetOrCreateDataNode("127.0.0.1", 8081, flakyPort, "127.0.0.1", "dn-flaky", map[string]uint32{"": 10})
|
||||
rp, err := super_block.NewReplicaPlacementFromString("001")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := storage.VolumeInfo{
|
||||
Id: needle.VolumeId(1),
|
||||
Collection: "c",
|
||||
ModifiedAtSecond: time.Now().Unix() - 7200,
|
||||
Version: needle.GetCurrentVersion(),
|
||||
ReplicaPlacement: rp,
|
||||
Ttl: needle.EMPTY_TTL,
|
||||
}
|
||||
for _, dn := range []*DataNode{okDn, flakyDn} {
|
||||
dn.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
topo.RegisterVolumeLayout(v, dn)
|
||||
}
|
||||
vl := topo.GetVolumeLayout("c", rp, needle.EMPTY_TTL, types.ToDiskType(""))
|
||||
|
||||
sweep := func() map[needle.VolumeId]*VolumeLocationList {
|
||||
vl.accessLock.RLock()
|
||||
todo := map[needle.VolumeId]*VolumeLocationList{}
|
||||
for vid, ll := range vl.vid2location {
|
||||
todo[vid] = ll.Copy()
|
||||
}
|
||||
vl.accessLock.RUnlock()
|
||||
topo.deleteEmptyVolumes(dialOption, vl, todo, time.Hour)
|
||||
return todo
|
||||
}
|
||||
|
||||
sweep()
|
||||
if !vl.IsMarkedDeleting(v.Id) {
|
||||
t.Fatal("partly deleted vid lost its deleting mark")
|
||||
}
|
||||
|
||||
readOnly := v
|
||||
readOnly.ReadOnly = true
|
||||
flakyDn.UpdateVolumes([]storage.VolumeInfo{readOnly})
|
||||
if todo := sweep(); todo[v.Id] == nil {
|
||||
t.Fatal("read-only survivor was deleted")
|
||||
}
|
||||
if !vl.IsMarkedDeleting(v.Id) {
|
||||
t.Fatal("read-only survivor dropped the deleting mark")
|
||||
}
|
||||
|
||||
flakyDn.UpdateVolumes([]storage.VolumeInfo{v})
|
||||
if todo := sweep(); todo[v.Id] != nil {
|
||||
t.Fatal("sweep left the vid behind once the survivor was deletable again")
|
||||
}
|
||||
flakyFake.mu.Lock()
|
||||
defer flakyFake.mu.Unlock()
|
||||
if len(flakyFake.deletes) != 1 {
|
||||
t.Fatalf("VolumeDelete on the survivor = %d, want 1 after it was deletable again", len(flakyFake.deletes))
|
||||
}
|
||||
}
|
||||
|
||||
// With every copy eligible a copy whose delete RPC fails keeps the vid in the
|
||||
// sweep map; the deleted copy is gone but the vid falls through to the normal
|
||||
// compaction path for its surviving replicas.
|
||||
|
||||
@@ -648,6 +648,13 @@ func (vl *VolumeLayout) MarkDeleting(vid needle.VolumeId) {
|
||||
vl.waitForPendingDrain(context.Background(), vid)
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) IsMarkedDeleting(vid needle.VolumeId) bool {
|
||||
vl.accessLock.RLock()
|
||||
defer vl.accessLock.RUnlock()
|
||||
_, ok := vl.deletingVolumes[vid]
|
||||
return ok
|
||||
}
|
||||
|
||||
func (vl *VolumeLayout) UnmarkDeleting(vid needle.VolumeId) {
|
||||
vl.accessLock.Lock()
|
||||
delete(vl.deletingVolumes, vid)
|
||||
|
||||
Reference in new issue
Block a user