Files
seaweedfs/seaweed-volume/src/metrics.rs
T
d6c65cd924 volume: count EC intervals served by reconstruction (#12392)
* volume: count EC reads that fall back to reconstruction

When a needle read can't read an EC shard interval directly (the shard isn't
mounted here and no holder answers), both volume servers rebuild it from the
other shards and the read still succeeds, with nothing to show for it.

Count those fallbacks in SeaweedFS_volumeServer_ec_degraded_read_total{result}
(success, failure), one per interval, at the one place each server falls back:
Go readOneEcShardInterval, Rust fetch_one_interval. Corruption repair and scrub
aren't counted. A line binding each one to the needle and the volume's encode
generation is logged at V(3) in Go and debug in Rust.

* volume: name the EC counter for what it counts

Reed-Solomon reconstruction can't tell corrupt inputs from good ones, and the
needle is validated later, so "success" only ever meant the reconstruction
produced data. Rename the counter to
SeaweedFS_volumeServer_ec_reconstructed_interval_total and the log line to
"ec reconstructed interval", say so in the help text, and pin the behaviour
with a test: an interval rebuilt from corrupt parity counts success while the
read itself fails.

* storage: restore stderr when the log-line test stops early

A fixture failure in readEcNeedle ends the test through t.Fatalf, which
skipped restoring os.Stderr and closing the pipe, so later tests wrote
into a pipe nobody read.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-04 08:30:41 -08:00

639 lines
23 KiB
Rust

//! Prometheus metrics for the volume server.
//!
//! Mirrors the Go SeaweedFS volume server metrics.
use prometheus::{
self, Encoder, GaugeVec, HistogramOpts, HistogramVec, IntCounter, IntCounterVec, IntGauge,
IntGaugeVec, Opts, Registry, TextEncoder,
};
use std::sync::{LazyLock, Once};
use crate::version;
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct PushGatewayConfig {
pub address: String,
pub interval_seconds: u32,
}
pub static REGISTRY: LazyLock<Registry> = LazyLock::new(Registry::new);
// ---- Request metrics (Go: VolumeServerRequestCounter, VolumeServerRequestHistogram) ----
/// Request counter with labels `type` (HTTP method) and `code` (HTTP status).
pub static REQUEST_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_request_total",
"Volume server requests",
),
&["type", "code"],
)
.expect("metric can be created")
});
/// Request duration histogram with label `type` (HTTP method).
pub static REQUEST_DURATION: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"SeaweedFS_volumeServer_request_seconds",
"Volume server request duration in seconds",
)
.buckets(exponential_buckets(0.0001, 2.0, 24)),
&["type"],
)
.expect("metric can be created")
});
// ---- Handler counters (Go: VolumeServerHandlerCounter) ----
/// Handler-level operation counter with label `type`.
pub static HANDLER_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_handler_total",
"Volume server handler counters",
),
&["type"],
)
.expect("metric can be created")
});
// ---- Vacuuming metrics (Go: VolumeServerVacuuming*) ----
/// Vacuuming compact counter with label `success` (true/false).
pub static VACUUMING_COMPACT_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_vacuuming_compact_count",
"Counter of volume vacuuming Compact counter",
),
&["success"],
)
.expect("metric can be created")
});
/// Vacuuming commit counter with label `success` (true/false).
pub static VACUUMING_COMMIT_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_vacuuming_commit_count",
"Counter of volume vacuuming commit counter",
),
&["success"],
)
.expect("metric can be created")
});
/// Vacuuming duration histogram with label `type` (compact/commit).
pub static VACUUMING_HISTOGRAM: LazyLock<HistogramVec> = LazyLock::new(|| {
HistogramVec::new(
HistogramOpts::new(
"SeaweedFS_volumeServer_vacuuming_seconds",
"Volume vacuuming duration in seconds",
)
.buckets(exponential_buckets(0.0001, 2.0, 24)),
&["type"],
)
.expect("metric can be created")
});
// ---- Volume gauges (Go: VolumeServerVolumeGauge, VolumeServerReadOnlyVolumeGauge) ----
/// Volumes per collection and type (volume/ec_shards).
pub static VOLUME_GAUGE: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new("SeaweedFS_volumeServer_volumes", "Number of volumes"),
&["collection", "type"],
)
.expect("metric can be created")
});
/// Read-only volumes per collection and type.
pub static READ_ONLY_VOLUME_GAUGE: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_read_only_volumes",
"Number of read-only volumes.",
),
&["collection", "type"],
)
.expect("metric can be created")
});
/// Maximum number of volumes this server can hold.
pub static MAX_VOLUMES: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"SeaweedFS_volumeServer_max_volumes",
"Maximum number of volumes",
)
.expect("metric can be created")
});
// ---- Disk size gauges (Go: VolumeServerDiskSizeGauge) ----
/// Actual disk size used by volumes per collection and type (normal/deleted_bytes/ec).
pub static DISK_SIZE_GAUGE: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_total_disk_size",
"Actual disk size used by volumes",
),
&["collection", "type"],
)
.expect("metric can be created")
});
// ---- Resource gauges (Go: VolumeServerResourceGauge) ----
/// Disk resource usage per directory and type (all/used/free/avail).
pub static RESOURCE_GAUGE: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new("SeaweedFS_volumeServer_resource", "Server resource usage"),
&["name", "type"],
)
.expect("metric can be created")
});
// ---- In-flight gauges (Go: VolumeServerInFlightRequestsGauge, InFlightDownload/UploadSize) ----
/// In-flight requests per HTTP method.
pub static INFLIGHT_REQUESTS_GAUGE: LazyLock<IntGaugeVec> = LazyLock::new(|| {
IntGaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_in_flight_requests",
"Current number of in-flight requests being handled by volume server.",
),
&["type"],
)
.expect("metric can be created")
});
/// Concurrent download limit in bytes.
pub static CONCURRENT_DOWNLOAD_LIMIT: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"SeaweedFS_volumeServer_concurrent_download_limit",
"Limit for total concurrent download size in bytes",
)
.expect("metric can be created")
});
/// Concurrent upload limit in bytes.
pub static CONCURRENT_UPLOAD_LIMIT: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"SeaweedFS_volumeServer_concurrent_upload_limit",
"Limit for total concurrent upload size in bytes",
)
.expect("metric can be created")
});
/// Current in-flight download bytes.
pub static INFLIGHT_DOWNLOAD_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"SeaweedFS_volumeServer_in_flight_download_size",
"In flight total download size.",
)
.expect("metric can be created")
});
/// Current in-flight upload bytes.
pub static INFLIGHT_UPLOAD_SIZE: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"SeaweedFS_volumeServer_in_flight_upload_size",
"In flight total upload size.",
)
.expect("metric can be created")
});
/// Upload error counter by HTTP status code. Code "0" = transport error (no response).
pub static UPLOAD_ERROR_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new("SeaweedFS_upload_error_total",
"Counter of upload errors by HTTP status code. Code 0 means transport error (no response received)."),
&["code"],
).expect("metric can be created")
});
// ---- Scrubbing metrics (Go: VolumeServerScrub*) ----
/// Last scrub execution time, as seconds since UNIX epoch, with label `mode`.
pub static SCRUB_LAST_TIME_SECONDS: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_last_time_seconds",
"Last scrub execution time, as seconds since UNIX epoch.",
),
&["mode"],
)
.expect("metric can be created")
});
/// Counter of overall volumes with issues detected during scrubbing, with label `mode`.
pub static SCRUB_VOLUME_FAILURES: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_volume_failures",
"Counter of overall volumes with issues detected during scrubbing.",
),
&["mode"],
)
.expect("metric can be created")
});
/// Counter of overall EC shards with issues detected during scrubbing, with label `mode`.
pub static SCRUB_SHARD_FAILURES: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_scrub_shard_failures",
"Counter of overall EC shards with issues detected during scrubbing.",
),
&["mode"],
)
.expect("metric can be created")
});
/// Reads of an EC shard interval that couldn't be read directly and were
/// served by reconstructing it from the other shards, by result. `success`
/// means the reconstruction produced data, not that the needle then passed
/// validation. Mirrors Go's VolumeServerECReconstructedIntervalCounter.
pub static EC_RECONSTRUCTED_INTERVAL_COUNTER: LazyLock<IntCounterVec> = LazyLock::new(|| {
IntCounterVec::new(
Opts::new(
"SeaweedFS_volumeServer_ec_reconstructed_interval_total",
"Reads of an EC interval served by reconstructing from other shards; success means reconstruction produced data, not that the needle passed validation.",
),
&["result"],
)
.expect("metric can be created")
});
/// Counter of storage read/write EIO errors on volumes and EC shards.
/// Mirrors Go's VolumeServerStorageIoErrorCounter.
pub static STORAGE_IO_ERROR_COUNTER: LazyLock<IntCounter> = LazyLock::new(|| {
IntCounter::new(
"SeaweedFS_volumeServer_storage_io_error_total",
"Counter of storage read/write EIO errors on volumes and EC shards.",
)
.expect("metric can be created")
});
/// Number of volumes quarantined due to storage IO errors.
/// Mirrors Go's VolumeServerIoQuarantineGauge.
pub static IO_QUARANTINE_GAUGE: LazyLock<IntGaugeVec> = LazyLock::new(|| {
IntGaugeVec::new(
Opts::new(
"SeaweedFS_volumeServer_io_quarantine",
"Number of volumes or EC shards quarantined due to storage IO errors.",
),
&["kind"],
)
.expect("metric can be created")
});
// ---- Legacy aliases for backward compat with existing code ----
/// Total number of volumes on this server (flat gauge).
pub static VOLUMES_TOTAL: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new("volume_server_volumes_total", "Total number of volumes")
.expect("metric can be created")
});
/// Disk size in bytes per directory.
pub static DISK_SIZE_BYTES: LazyLock<IntGaugeVec> = LazyLock::new(|| {
IntGaugeVec::new(
Opts::new("volume_server_disk_size_bytes", "Disk size in bytes"),
&["dir"],
)
.expect("metric can be created")
});
/// Disk free bytes per directory.
pub static DISK_FREE_BYTES: LazyLock<IntGaugeVec> = LazyLock::new(|| {
IntGaugeVec::new(
Opts::new("volume_server_disk_free_bytes", "Disk free space in bytes"),
&["dir"],
)
.expect("metric can be created")
});
/// Current number of in-flight requests (flat gauge).
pub static INFLIGHT_REQUESTS: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"volume_server_inflight_requests",
"Current number of in-flight requests",
)
.expect("metric can be created")
});
/// Total number of files stored across all volumes.
pub static VOLUME_FILE_COUNT: LazyLock<IntGauge> = LazyLock::new(|| {
IntGauge::new(
"volume_server_volume_file_count",
"Total number of files stored across all volumes",
)
.expect("metric can be created")
});
// ---- Build info (Go: BuildInfo) ----
/// Build information gauge, always set to 1. Matches Go:
/// Namespace="SeaweedFS", Subsystem="build", Name="info",
/// labels: version, commit, sizelimit, goos, goarch.
pub static BUILD_INFO: LazyLock<GaugeVec> = LazyLock::new(|| {
GaugeVec::new(
Opts::new("SeaweedFS_build_info", "A metric with a constant '1' value labeled by version, commit, sizelimit, goos, and goarch from which SeaweedFS was built."),
&["version", "commit", "sizelimit", "goos", "goarch"],
).expect("metric can be created")
});
/// Generate exponential bucket boundaries for histograms.
fn exponential_buckets(start: f64, factor: f64, count: usize) -> Vec<f64> {
let mut buckets = Vec::with_capacity(count);
let mut val = start;
for _ in 0..count {
buckets.push(val);
val *= factor;
}
buckets
}
// Handler counter type constants (matches Go's metrics_names.go).
pub const WRITE_TO_LOCAL_DISK: &str = "writeToLocalDisk";
pub const WRITE_TO_REPLICAS: &str = "writeToReplicas";
pub const DOWNLOAD_LIMIT_COND: &str = "downloadLimitCondition";
pub const UPLOAD_LIMIT_COND: &str = "uploadLimitCondition";
pub const READ_PROXY_REQ: &str = "readProxyRequest";
pub const READ_REDIRECT_REQ: &str = "readRedirectRequest";
pub const READ_DELETED_NEEDLE: &str = "readDeletedNeedle";
pub const EMPTY_READ_PROXY_LOC: &str = "emptyReadProxyLocaction";
pub const FAILED_READ_PROXY_REQ: &str = "failedReadProxyRequest";
// Error metric name constants.
pub const ERROR_SIZE_MISMATCH_OFFSET_SIZE: &str = "errorSizeMismatchOffsetSize";
pub const ERROR_SIZE_MISMATCH: &str = "errorSizeMismatch";
pub const ERROR_CRC: &str = "errorCRC";
pub const ERROR_INDEX_OUT_OF_RANGE: &str = "errorIndexOutOfRange";
pub const ERROR_GET_NOT_FOUND: &str = "errorGetNotFound";
pub const ERROR_GET_INTERNAL: &str = "errorGetInternal";
pub const ERROR_WRITE_TO_LOCAL_DISK: &str = "errorWriteToLocalDisk";
pub const ERROR_UNMARSHAL_PAIRS: &str = "errorUnmarshalPairs";
pub const ERROR_WRITE_TO_REPLICAS: &str = "errorWriteToReplicas";
// Go volume heartbeat metric label values.
pub const READ_ONLY_LABEL_IS_READ_ONLY: &str = "IsReadOnly";
pub const READ_ONLY_LABEL_NO_WRITE_OR_DELETE: &str = "noWriteOrDelete";
pub const READ_ONLY_LABEL_NO_WRITE_CAN_DELETE: &str = "noWriteCanDelete";
pub const READ_ONLY_LABEL_IS_DISK_SPACE_LOW: &str = "isDiskSpaceLow";
pub const DISK_SIZE_LABEL_NORMAL: &str = "normal";
pub const DISK_SIZE_LABEL_DELETED_BYTES: &str = "deleted_bytes";
pub const DISK_SIZE_LABEL_EC: &str = "ec";
static REGISTER_METRICS: Once = Once::new();
/// Register all metrics with the custom registry.
/// Call this once at startup.
pub fn register_metrics() {
REGISTER_METRICS.call_once(|| {
let metrics: Vec<Box<dyn prometheus::core::Collector>> = vec![
// New Go-compatible metrics
Box::new(REQUEST_COUNTER.clone()),
Box::new(REQUEST_DURATION.clone()),
Box::new(HANDLER_COUNTER.clone()),
Box::new(VACUUMING_COMPACT_COUNTER.clone()),
Box::new(VACUUMING_COMMIT_COUNTER.clone()),
Box::new(VACUUMING_HISTOGRAM.clone()),
Box::new(VOLUME_GAUGE.clone()),
Box::new(READ_ONLY_VOLUME_GAUGE.clone()),
Box::new(MAX_VOLUMES.clone()),
Box::new(DISK_SIZE_GAUGE.clone()),
Box::new(RESOURCE_GAUGE.clone()),
Box::new(INFLIGHT_REQUESTS_GAUGE.clone()),
Box::new(CONCURRENT_DOWNLOAD_LIMIT.clone()),
Box::new(CONCURRENT_UPLOAD_LIMIT.clone()),
Box::new(INFLIGHT_DOWNLOAD_SIZE.clone()),
Box::new(INFLIGHT_UPLOAD_SIZE.clone()),
Box::new(UPLOAD_ERROR_COUNTER.clone()),
Box::new(SCRUB_LAST_TIME_SECONDS.clone()),
Box::new(SCRUB_VOLUME_FAILURES.clone()),
Box::new(SCRUB_SHARD_FAILURES.clone()),
Box::new(STORAGE_IO_ERROR_COUNTER.clone()),
Box::new(EC_RECONSTRUCTED_INTERVAL_COUNTER.clone()),
Box::new(IO_QUARANTINE_GAUGE.clone()),
// Legacy metrics
Box::new(VOLUMES_TOTAL.clone()),
Box::new(DISK_SIZE_BYTES.clone()),
Box::new(DISK_FREE_BYTES.clone()),
Box::new(INFLIGHT_REQUESTS.clone()),
Box::new(VOLUME_FILE_COUNT.clone()),
// Build info
Box::new(BUILD_INFO.clone()),
];
for m in metrics {
REGISTRY.register(m).expect("metric registered");
}
// Set build info gauge to 1 with version/commit/sizelimit/os/arch labels (matches Go).
BUILD_INFO
.with_label_values(&[
version::version(),
version::commit(),
version::size_limit(),
std::env::consts::OS,
std::env::consts::ARCH,
])
.set(1.0);
});
}
/// Gather all metrics and encode them in Prometheus text exposition format.
pub fn gather_metrics() -> String {
let encoder = TextEncoder::new();
let metric_families = REGISTRY.gather();
let mut buffer = Vec::new();
encoder
.encode(&metric_families, &mut buffer)
.expect("encoding metrics");
String::from_utf8(buffer).expect("metrics are valid UTF-8")
}
pub fn delete_collection_metrics(collection: &str) {
// Mirrors Go's DeletePartialMatch(prometheus.Labels{"collection": collection})
// which removes ALL metric entries matching the collection label, regardless
// of other label values (like "type"). We gather the metric families to discover
// all type values dynamically, matching Go's partial-match behavior.
delete_partial_match_collection(&VOLUME_GAUGE, collection);
delete_partial_match_collection(&READ_ONLY_VOLUME_GAUGE, collection);
delete_partial_match_collection(&DISK_SIZE_GAUGE, collection);
}
/// Drop a collection's volume server series once its last volume leaves this
/// server. These gauges are only ever set for collections still present, so the
/// values from the heartbeat that saw the last volume would otherwise stand
/// until the process restarts.
pub fn delete_volume_server_collection_metrics(collection: &str) {
let _ = DISK_SIZE_GAUGE.remove_label_values(&[collection, DISK_SIZE_LABEL_NORMAL]);
let _ = DISK_SIZE_GAUGE.remove_label_values(&[collection, DISK_SIZE_LABEL_DELETED_BYTES]);
delete_partial_match_collection(&READ_ONLY_VOLUME_GAUGE, collection);
}
/// Remove all metric entries from a GaugeVec where the "collection" label matches.
/// This emulates Go's `DeletePartialMatch(prometheus.Labels{"collection": collection})`.
fn delete_partial_match_collection(gauge: &GaugeVec, collection: &str) {
use prometheus::core::Collector;
let families = gauge.collect();
for family in &families {
for metric in family.get_metric() {
let labels = metric.get_label();
let mut matches_collection = false;
let mut type_value = None;
for label in labels {
if label.get_name() == "collection" && label.get_value() == collection {
matches_collection = true;
}
if label.get_name() == "type" {
type_value = Some(label.get_value().to_string());
}
}
if matches_collection && let Some(ref tv) = type_value {
let _ = gauge.remove_label_values(&[collection, tv]);
}
}
}
}
pub fn build_pushgateway_url(address: &str, job: &str, instance: &str) -> String {
let base = if address.starts_with("http://") || address.starts_with("https://") {
address.to_string()
} else {
format!("http://{}", address)
};
let base = base.trim_end_matches('/');
format!("{}/metrics/job/{}/instance/{}", base, job, instance)
}
pub async fn push_metrics_once(
client: &reqwest::Client,
address: &str,
job: &str,
instance: &str,
) -> Result<(), String> {
let url = build_pushgateway_url(address, job, instance);
let response = client
.put(&url)
.header(
reqwest::header::CONTENT_TYPE,
"text/plain; version=0.0.4; charset=utf-8",
)
.body(gather_metrics())
.send()
.await
.map_err(|e| format!("push metrics request failed: {}", e))?;
if response.status().is_success() {
Ok(())
} else {
Err(format!(
"push metrics failed with status {}",
response.status()
))
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::{Router, routing::put};
use std::sync::{Arc, Mutex};
#[test]
fn test_gather_metrics_returns_text() {
register_metrics();
REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
let output = gather_metrics();
assert!(output.contains("SeaweedFS_volumeServer_request_total"));
}
#[test]
fn test_build_pushgateway_url() {
assert_eq!(
build_pushgateway_url("localhost:9091", "volumeServer", "test-instance"),
"http://localhost:9091/metrics/job/volumeServer/instance/test-instance"
);
assert_eq!(
build_pushgateway_url("https://push.example", "volumeServer", "node-a"),
"https://push.example/metrics/job/volumeServer/instance/node-a"
);
}
#[tokio::test]
async fn test_push_metrics_once() {
register_metrics();
// A CounterVec with no children emits nothing, so create a labelset
// instead of depending on another test having touched the counter
// first (test order is nondeterministic).
REQUEST_COUNTER.with_label_values(&["GET", "200"]).inc();
let captured = Arc::new(Mutex::new(None::<String>));
let captured_clone = captured.clone();
let app = Router::new().route(
"/metrics/job/volumeServer/instance/test-instance",
put(move |body: String| {
let captured = captured_clone.clone();
async move {
*captured.lock().unwrap() = Some(body);
"ok"
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let server = tokio::spawn(async move {
axum::serve(listener, app).await.unwrap();
});
let client = reqwest::Client::new();
push_metrics_once(
&client,
&format!("127.0.0.1:{}", addr.port()),
"volumeServer",
"test-instance",
)
.await
.unwrap();
let body = captured.lock().unwrap().clone().unwrap();
assert!(body.contains("SeaweedFS_volumeServer_request_total"));
server.abort();
}
#[test]
fn test_delete_collection_metrics_removes_collection_labelsets() {
register_metrics();
VOLUME_GAUGE.with_label_values(&["pics", "volume"]).set(2.0);
VOLUME_GAUGE
.with_label_values(&["pics", "ec_shards"])
.set(3.0);
READ_ONLY_VOLUME_GAUGE
.with_label_values(&["pics", "volume"])
.set(1.0);
DISK_SIZE_GAUGE
.with_label_values(&["pics", "normal"])
.set(10.0);
DISK_SIZE_GAUGE
.with_label_values(&["pics", "deleted_bytes"])
.set(4.0);
delete_collection_metrics("pics");
let output = gather_metrics();
assert!(!output.contains("collection=\"pics\",type=\"volume\""));
assert!(!output.contains("collection=\"pics\",type=\"ec_shards\""));
assert!(!output.contains("collection=\"pics\",type=\"normal\""));
assert!(!output.contains("collection=\"pics\",type=\"deleted_bytes\""));
}
}