diff --git a/weed/admin/dash/ec_shard_management.go b/weed/admin/dash/ec_shard_management.go index f9caf306b2..00ca29bcf8 100644 --- a/weed/admin/dash/ec_shard_management.go +++ b/weed/admin/dash/ec_shard_management.go @@ -915,26 +915,61 @@ func (s *AdminServer) getVolumeHealthFromServer(server string, volumeID uint32) var healthInfo *EcVolumeHealthInfo err := s.WithVolumeServerClient(pb.ServerAddress(server), func(client volume_server_pb.VolumeServerClient) error { + var collection string = "" // Default collection name + var totalSize uint64 = 0 + var fileCount uint64 = 0 + // Try to get volume file status (which may include original volume metrics) + // This will fail for EC-only volumes, so we handle that gracefully resp, err := client.ReadVolumeFileStatus(context.Background(), &volume_server_pb.ReadVolumeFileStatusRequest{ VolumeId: volumeID, }) if err != nil { - return err + glog.V(2).Infof("ReadVolumeFileStatus failed for EC volume %d on server %s (expected for EC-only volumes): %v", volumeID, server, err) + // For EC-only volumes, we don't have original volume metrics, but we can still get deletion info + } else if resp.VolumeInfo != nil { + // Extract metrics from regular volume info if available + totalSize = uint64(resp.VolumeInfo.DatFileSize) + fileCount = resp.FileCount + collection = resp.Collection } - // Extract health metrics from volume info - if resp.VolumeInfo != nil { - totalSize := uint64(resp.VolumeInfo.DatFileSize) + // Always try to get EC deletion information using the new gRPC endpoint + deletionResp, deletionErr := client.VolumeEcDeletionInfo(context.Background(), &volume_server_pb.VolumeEcDeletionInfoRequest{ + VolumeId: volumeID, + Collection: collection, + Generation: 0, // Use default generation for backward compatibility + }) + + if deletionErr != nil { + glog.V(1).Infof("Failed to get EC deletion info for volume %d on server %s: %v", volumeID, server, deletionErr) + // If we have some info from ReadVolumeFileStatus, still create healthInfo with that if totalSize > 0 { healthInfo = &EcVolumeHealthInfo{ TotalSize: totalSize, - DeletedByteCount: 0, // EC volumes don't track deletions in VolumeInfo - FileCount: resp.FileCount, - DeleteCount: 0, // Not available in current API + DeletedByteCount: 0, + FileCount: fileCount, + DeleteCount: 0, GarbageRatio: 0.0, } } + } else if deletionResp != nil { + // Create health info with deletion data + healthInfo = &EcVolumeHealthInfo{ + TotalSize: totalSize, // May be 0 for EC-only volumes + DeletedByteCount: deletionResp.DeletedBytes, + FileCount: fileCount, + DeleteCount: deletionResp.DeletedCount, + GarbageRatio: 0.0, + } + + // Calculate garbage ratio if we have total size + if healthInfo.TotalSize > 0 { + healthInfo.GarbageRatio = float64(healthInfo.DeletedByteCount) / float64(healthInfo.TotalSize) + } + + glog.V(1).Infof("EC volume %d on server %s: %d deleted bytes, %d deleted needles, total size: %d bytes", + volumeID, server, healthInfo.DeletedByteCount, healthInfo.DeleteCount, healthInfo.TotalSize) } return nil diff --git a/weed/admin/maintenance/maintenance_scanner.go b/weed/admin/maintenance/maintenance_scanner.go index 9b130bf170..3b83c7cb48 100644 --- a/weed/admin/maintenance/maintenance_scanner.go +++ b/weed/admin/maintenance/maintenance_scanner.go @@ -6,9 +6,14 @@ import ( "time" "github.com/seaweedfs/seaweedfs/weed/glog" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding" "github.com/seaweedfs/seaweedfs/weed/worker/types" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) // NewMaintenanceScanner creates a new maintenance scanner @@ -477,35 +482,28 @@ func (ms *MaintenanceScanner) collectECVolumeDelationsFromAllServers(volumeId ui } // Merge deletion information, avoiding double counting + newNeedles := 0 for needle := range serverDeletedNeedles { if !deletedNeedles[needle] { deletedNeedles[needle] = true - // We can't get exact size per needle without more complex analysis, - // so we'll use the server's reported total as a conservative estimate - // This could be enhanced with proper .ecj parsing + newNeedles++ } } - // For now, sum the deleted bytes from all servers - // Note: This might double-count if the same needle is deleted across shards, - // but it provides a reasonable upper bound estimate - totalDeletedBytes += serverDeletedBytes - - glog.V(3).Infof("Server %s reported %d deleted bytes for EC volume %d", server, serverDeletedBytes, volumeId) - } - - // Apply conservative adjustment to account for potential double counting - // Since deletions are tracked per shard but affect the whole needle, - // we should not simply sum all deleted bytes from all servers - if len(servers) > 1 && totalDeletedBytes > 0 { - // Conservative approach: assume some overlap and reduce the total - adjustmentFactor := float64(len(deletedNeedles)) / float64(len(servers)) - if adjustmentFactor < 1.0 { - totalDeletedBytes = int64(float64(totalDeletedBytes) * adjustmentFactor) - glog.V(3).Infof("Applied conservative adjustment factor %.2f to EC volume %d deleted bytes", adjustmentFactor, volumeId) + // Only add bytes for needles that are actually new to avoid double counting + // Assume bytes are evenly distributed per needle for deduplication + if len(serverDeletedNeedles) > 0 && serverDeletedBytes > 0 { + avgBytesPerNeedle := float64(serverDeletedBytes) / float64(len(serverDeletedNeedles)) + totalDeletedBytes += int64(float64(newNeedles) * avgBytesPerNeedle) } + + glog.V(3).Infof("Server %s reported %d deleted bytes, %d needles (%d new) for EC volume %d", + server, serverDeletedBytes, len(serverDeletedNeedles), newNeedles, volumeId) } + glog.V(2).Infof("EC volume %d total: %d unique deleted needles, %d estimated deleted bytes from %d servers", + volumeId, len(deletedNeedles), totalDeletedBytes, len(servers)) + return totalDeletedBytes, nil } @@ -557,17 +555,46 @@ func (ms *MaintenanceScanner) getServerECVolumeDeletions(volumeId uint32, collec // return totalDeleted, deletedNeedleMap, nil // }) - // For now, implement a conservative heuristic approach + // Use the new VolumeEcDeletionInfo gRPC endpoint to get accurate deletion data + var deletedBytes int64 = 0 deletedNeedles := make(map[string]bool) - // Very conservative estimate to avoid false positives - // This will be replaced with proper .ecj/.ecx analysis - conservativeEstimate := int64(1024) // 1KB conservative estimate per server + glog.V(0).Infof("Making gRPC call to server %s for volume %d collection %s", server, volumeId, collection) - glog.V(4).Infof("Applied conservative deletion estimate for EC volume %d on server %s: %d bytes (heuristic mode)", - volumeId, server, conservativeEstimate) + err := operation.WithVolumeServerClient(false, pb.ServerAddress(server), + grpc.WithTransportCredentials(insecure.NewCredentials()), func(client volume_server_pb.VolumeServerClient) error { + glog.V(0).Infof("Connected to volume server %s, calling VolumeEcDeletionInfo", server) + resp, err := client.VolumeEcDeletionInfo(context.Background(), &volume_server_pb.VolumeEcDeletionInfoRequest{ + VolumeId: volumeId, + Collection: collection, + Generation: 0, // Use default generation for backward compatibility + }) + if err != nil { + glog.V(0).Infof("VolumeEcDeletionInfo call failed for server %s: %v", server, err) + return err + } - return conservativeEstimate, deletedNeedles, nil + deletedBytes = int64(resp.DeletedBytes) + + // Convert deleted needle IDs to map for duplicate tracking + for _, needleId := range resp.DeletedNeedleIds { + deletedNeedles[fmt.Sprintf("%d", needleId)] = true + } + + glog.V(0).Infof("Got EC deletion info for volume %d on server %s: %d bytes, %d needles", + volumeId, server, deletedBytes, len(resp.DeletedNeedleIds)) + + return nil + }) + + if err != nil { + glog.V(0).Infof("Failed to get EC deletion info for volume %d on server %s, using conservative estimate: %v", volumeId, server, err) + // Fallback to conservative estimate if gRPC call fails + deletedBytes = int64(1024) // 1KB conservative estimate + } + + glog.V(0).Infof("Returning from getServerECVolumeDeletions: %d bytes, %d needles", deletedBytes, len(deletedNeedles)) + return deletedBytes, deletedNeedles, nil } // convertToTaskMetrics converts existing volume metrics to task system format diff --git a/weed/admin/view/app/ec_volume_details.templ b/weed/admin/view/app/ec_volume_details.templ index c0fb32187e..811c17116f 100644 --- a/weed/admin/view/app/ec_volume_details.templ +++ b/weed/admin/view/app/ec_volume_details.templ @@ -152,64 +152,102 @@ templ EcVolumeDetails(data dash.EcVolumeDetailsData) {
This volume may not be EC encoded yet.
This volume may not be EC encoded yet.