mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-08 15:41:15 +02:00
feat(filer): object size distribution metric and dashboard panels (#9902)
* feat(filer): record object size distribution histogram Add SeaweedFS_filer_object_size_bytes, a histogram sampled when an object is first created in the filer namespace, covering every write protocol (S3, WebDAV, FUSE mount, direct HTTP). Buckets follow the 1KB/100KB/1MB/100MB/1GB ranges operators use to size collections. Directories, overwrites, and metadata-only updates are not sampled, so the bucket counts track the size distribution of distinct objects. * feat(metrics): add filer object size distribution dashboard panels Add a write-rate-by-size-range graph and a size-distribution bar gauge, driven by SeaweedFS_filer_object_size_bytes, to the standalone and Helm Grafana dashboards. Per-range subtractions are clamped at zero so transient negative rate() samples do not render below the axis.
This commit is contained in:
@@ -3666,6 +3666,291 @@
|
|||||||
],
|
],
|
||||||
"title": "S3 Bucket Object Count",
|
"title": "S3 Bucket Object Count",
|
||||||
"type": "timeseries"
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"collapsed": false,
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "${DS_PROMETHEUS}"
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 1,
|
||||||
|
"w": 24,
|
||||||
|
"x": 0,
|
||||||
|
"y": 130
|
||||||
|
},
|
||||||
|
"id": 92,
|
||||||
|
"panels": [],
|
||||||
|
"title": "Filer Object Size Distribution",
|
||||||
|
"type": "row"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "${DS_PROMETHEUS}"
|
||||||
|
},
|
||||||
|
"description": "Rate of new objects created, split by size range.",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "palette-classic"
|
||||||
|
},
|
||||||
|
"custom": {
|
||||||
|
"axisBorderShow": false,
|
||||||
|
"axisCenteredZero": false,
|
||||||
|
"axisColorMode": "text",
|
||||||
|
"axisLabel": "objects/s",
|
||||||
|
"axisPlacement": "auto",
|
||||||
|
"barAlignment": 0,
|
||||||
|
"drawStyle": "line",
|
||||||
|
"fillOpacity": 25,
|
||||||
|
"gradientMode": "none",
|
||||||
|
"hideFrom": {
|
||||||
|
"legend": false,
|
||||||
|
"tooltip": false,
|
||||||
|
"viz": false
|
||||||
|
},
|
||||||
|
"insertNulls": false,
|
||||||
|
"lineInterpolation": "linear",
|
||||||
|
"lineWidth": 1,
|
||||||
|
"pointSize": 4,
|
||||||
|
"scaleDistribution": {
|
||||||
|
"type": "linear"
|
||||||
|
},
|
||||||
|
"showPoints": "auto",
|
||||||
|
"spanNulls": false,
|
||||||
|
"stacking": {
|
||||||
|
"group": "A",
|
||||||
|
"mode": "normal"
|
||||||
|
},
|
||||||
|
"thresholdsStyle": {
|
||||||
|
"mode": "off"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "short",
|
||||||
|
"unitScale": true
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 131
|
||||||
|
},
|
||||||
|
"id": 93,
|
||||||
|
"links": [],
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [],
|
||||||
|
"displayMode": "list",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "single",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"pluginVersion": "8.1.2",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval]))",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "< 1KB",
|
||||||
|
"refId": "A",
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1KB - 100KB",
|
||||||
|
"refId": "B",
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100KB - 1MB",
|
||||||
|
"refId": "C",
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1MB - 100MB",
|
||||||
|
"refId": "D",
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100MB - 1GB",
|
||||||
|
"refId": "E",
|
||||||
|
"step": 60
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__rate_interval])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__rate_interval])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"hide": false,
|
||||||
|
"instant": false,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "> 1GB",
|
||||||
|
"refId": "F",
|
||||||
|
"step": 60
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Filer Object Write Rate by Size Range",
|
||||||
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "${DS_PROMETHEUS}"
|
||||||
|
},
|
||||||
|
"description": "Objects created per size range over the selected time window.",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "thresholds"
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"min": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 8,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 131
|
||||||
|
},
|
||||||
|
"id": 94,
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"showUnfilled": true,
|
||||||
|
"valueMode": "color"
|
||||||
|
},
|
||||||
|
"pluginVersion": "8.1.2",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range]))",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "< 1KB",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1KB - 100KB",
|
||||||
|
"refId": "B"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100KB - 1MB",
|
||||||
|
"refId": "C"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1MB - 100MB",
|
||||||
|
"refId": "D"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100MB - 1GB",
|
||||||
|
"refId": "E"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"exemplar": true,
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count{namespace=\"$NAMESPACE\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\",namespace=\"$NAMESPACE\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"instant": true,
|
||||||
|
"interval": "",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "> 1GB",
|
||||||
|
"refId": "F"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Filer Object Size Distribution",
|
||||||
|
"type": "bargauge"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "",
|
"refresh": "",
|
||||||
|
|||||||
@@ -623,6 +623,237 @@
|
|||||||
"show": true
|
"show": true
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"aliasColors": {},
|
||||||
|
"bars": false,
|
||||||
|
"dashLength": 10,
|
||||||
|
"dashes": false,
|
||||||
|
"datasource": "${DS_PROMETHEUS}",
|
||||||
|
"editable": true,
|
||||||
|
"error": false,
|
||||||
|
"fill": 5,
|
||||||
|
"fillGradient": 0,
|
||||||
|
"grid": {},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 7,
|
||||||
|
"w": 12,
|
||||||
|
"x": 0,
|
||||||
|
"y": 31
|
||||||
|
},
|
||||||
|
"hiddenSeries": false,
|
||||||
|
"id": 91,
|
||||||
|
"legend": {
|
||||||
|
"alignAsTable": true,
|
||||||
|
"avg": true,
|
||||||
|
"current": true,
|
||||||
|
"hideEmpty": false,
|
||||||
|
"hideZero": false,
|
||||||
|
"max": true,
|
||||||
|
"min": false,
|
||||||
|
"rightSide": true,
|
||||||
|
"show": true,
|
||||||
|
"total": false,
|
||||||
|
"values": true
|
||||||
|
},
|
||||||
|
"lines": true,
|
||||||
|
"linewidth": 2,
|
||||||
|
"links": [],
|
||||||
|
"nullPointMode": "null as zero",
|
||||||
|
"options": {
|
||||||
|
"alertThreshold": true
|
||||||
|
},
|
||||||
|
"percentage": false,
|
||||||
|
"pluginVersion": "8.1.2",
|
||||||
|
"pointradius": 5,
|
||||||
|
"points": false,
|
||||||
|
"renderer": "flot",
|
||||||
|
"seriesOverrides": [],
|
||||||
|
"spaceLength": 10,
|
||||||
|
"stack": true,
|
||||||
|
"steppedLine": false,
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m]))",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "< 1KB",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[1m])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1KB - 100KB",
|
||||||
|
"refId": "B"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[1m])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100KB - 1MB",
|
||||||
|
"refId": "C"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[1m])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1MB - 100MB",
|
||||||
|
"refId": "D"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[1m])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100MB - 1GB",
|
||||||
|
"refId": "E"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(rate(SeaweedFS_filer_object_size_bytes_count[1m])) - sum(rate(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[1m])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "> 1GB",
|
||||||
|
"refId": "F"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"thresholds": [],
|
||||||
|
"timeFrom": null,
|
||||||
|
"timeRegions": [],
|
||||||
|
"timeShift": null,
|
||||||
|
"title": "Filer Object Write Rate by Size Range",
|
||||||
|
"tooltip": {
|
||||||
|
"msResolution": false,
|
||||||
|
"shared": true,
|
||||||
|
"sort": 0,
|
||||||
|
"value_type": "individual"
|
||||||
|
},
|
||||||
|
"type": "graph",
|
||||||
|
"xaxis": {
|
||||||
|
"buckets": null,
|
||||||
|
"mode": "time",
|
||||||
|
"name": null,
|
||||||
|
"show": true,
|
||||||
|
"values": []
|
||||||
|
},
|
||||||
|
"yaxes": [
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": "objects/s",
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": 0,
|
||||||
|
"show": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"format": "short",
|
||||||
|
"label": null,
|
||||||
|
"logBase": 1,
|
||||||
|
"max": null,
|
||||||
|
"min": null,
|
||||||
|
"show": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"yaxis": {
|
||||||
|
"align": false,
|
||||||
|
"alignLevel": null
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": "${DS_PROMETHEUS}",
|
||||||
|
"description": "Objects created per size range over the selected time window.",
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"color": {
|
||||||
|
"mode": "thresholds"
|
||||||
|
},
|
||||||
|
"mappings": [],
|
||||||
|
"min": 0,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"unit": "short"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"h": 7,
|
||||||
|
"w": 12,
|
||||||
|
"x": 12,
|
||||||
|
"y": 31
|
||||||
|
},
|
||||||
|
"id": 92,
|
||||||
|
"options": {
|
||||||
|
"displayMode": "gradient",
|
||||||
|
"orientation": "horizontal",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"showUnfilled": true,
|
||||||
|
"valueMode": "color"
|
||||||
|
},
|
||||||
|
"pluginVersion": "8.1.2",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range]))",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "< 1KB",
|
||||||
|
"refId": "A",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1024\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1KB - 100KB",
|
||||||
|
"refId": "B",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"102400\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100KB - 1MB",
|
||||||
|
"refId": "C",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+06\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "1MB - 100MB",
|
||||||
|
"refId": "D",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.048576e+08\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "100MB - 1GB",
|
||||||
|
"refId": "E",
|
||||||
|
"instant": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "clamp_min(sum(increase(SeaweedFS_filer_object_size_bytes_count[$__range])) - sum(increase(SeaweedFS_filer_object_size_bytes_bucket{le=\"1.073741824e+09\"}[$__range])), 0)",
|
||||||
|
"format": "time_series",
|
||||||
|
"intervalFactor": 2,
|
||||||
|
"legendFormat": "> 1GB",
|
||||||
|
"refId": "F",
|
||||||
|
"instant": true
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Filer Object Size Distribution",
|
||||||
|
"type": "bargauge"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"repeat": null,
|
"repeat": null,
|
||||||
|
|||||||
@@ -22,6 +22,7 @@ import (
|
|||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||||
@@ -256,6 +257,9 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, existing *Entry,
|
|||||||
glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err)
|
glog.ErrorfCtx(ctx, "insert entry %s: %v", entry.FullPath, err)
|
||||||
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
|
return fmt.Errorf("insert entry %s: %v", entry.FullPath, err)
|
||||||
}
|
}
|
||||||
|
if !entry.IsDirectory() {
|
||||||
|
stats.FilerObjectSizeBytesHistogram.Observe(float64(entry.Size()))
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
if o_excl {
|
if o_excl {
|
||||||
glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath)
|
glog.V(3).InfofCtx(ctx, "EEXIST: entry %s already exists", entry.FullPath)
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
package leveldb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
dto "github.com/prometheus/client_model/go"
|
||||||
|
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
func histogramState(t *testing.T) (count uint64, perBucket map[float64]uint64) {
|
||||||
|
t.Helper()
|
||||||
|
m := &dto.Metric{}
|
||||||
|
if err := stats.FilerObjectSizeBytesHistogram.Write(m); err != nil {
|
||||||
|
t.Fatalf("write histogram: %v", err)
|
||||||
|
}
|
||||||
|
perBucket = make(map[float64]uint64)
|
||||||
|
for _, b := range m.GetHistogram().GetBucket() {
|
||||||
|
perBucket[b.GetUpperBound()] = b.GetCumulativeCount()
|
||||||
|
}
|
||||||
|
return m.GetHistogram().GetSampleCount(), perBucket
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCreateEntryRecordsObjectSize(t *testing.T) {
|
||||||
|
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||||
|
store := &LevelDBStore{}
|
||||||
|
if err := store.initialize(t.TempDir()); err != nil {
|
||||||
|
t.Fatalf("init store: %v", err)
|
||||||
|
}
|
||||||
|
testFiler.SetStore(store)
|
||||||
|
defer testFiler.Shutdown()
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
file := func(path string, size uint64) *filer.Entry {
|
||||||
|
return &filer.Entry{
|
||||||
|
FullPath: util.FullPath(path),
|
||||||
|
Attr: filer.Attr{Mode: 0640, FileSize: size},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
before, _ := histogramState(t)
|
||||||
|
|
||||||
|
if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 500), nil, false, false, nil, false, 255); err != nil {
|
||||||
|
t.Fatalf("create small: %v", err)
|
||||||
|
}
|
||||||
|
if err := testFiler.CreateEntry(ctx, file("/data/big.bin", 5*1024*1024), nil, false, false, nil, false, 255); err != nil {
|
||||||
|
t.Fatalf("create big: %v", err)
|
||||||
|
}
|
||||||
|
dir := &filer.Entry{FullPath: util.FullPath("/data/sub"), Attr: filer.Attr{Mode: os.ModeDir | 0755}}
|
||||||
|
if err := testFiler.CreateEntry(ctx, dir, nil, false, false, nil, false, 255); err != nil {
|
||||||
|
t.Fatalf("create dir: %v", err)
|
||||||
|
}
|
||||||
|
// overwrite is an update, not a new object
|
||||||
|
if err := testFiler.CreateEntry(ctx, file("/data/small.txt", 800), nil, false, false, nil, false, 255); err != nil {
|
||||||
|
t.Fatalf("overwrite small: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
after, buckets := histogramState(t)
|
||||||
|
|
||||||
|
// only the two new files are sampled, into their respective ranges
|
||||||
|
if got := after - before; got != 2 {
|
||||||
|
t.Fatalf("expected 2 sampled objects, got %d", got)
|
||||||
|
}
|
||||||
|
if buckets[1024] < 1 {
|
||||||
|
t.Errorf("expected the 500-byte object in the <=1024 bucket, buckets=%v", buckets)
|
||||||
|
}
|
||||||
|
if buckets[104857600]-buckets[1048576] < 1 {
|
||||||
|
t.Errorf("expected the 5MB object in the (1MB,100MB] range, buckets=%v", buckets)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -199,6 +199,16 @@ var (
|
|||||||
Help: "The last send timestamp of the filer subscription.",
|
Help: "The last send timestamp of the filer subscription.",
|
||||||
}, []string{"sourceFiler", "clientName", "path"})
|
}, []string{"sourceFiler", "clientName", "path"})
|
||||||
|
|
||||||
|
// Sampled only on first creation, so counts track distinct objects.
|
||||||
|
FilerObjectSizeBytesHistogram = prometheus.NewHistogram(
|
||||||
|
prometheus.HistogramOpts{
|
||||||
|
Namespace: Namespace,
|
||||||
|
Subsystem: "filer",
|
||||||
|
Name: "object_size_bytes",
|
||||||
|
Help: "Distribution of object sizes in bytes, sampled when an object is first created.",
|
||||||
|
Buckets: []float64{1024, 102400, 1048576, 104857600, 1073741824},
|
||||||
|
})
|
||||||
|
|
||||||
FilerStoreCounter = prometheus.NewCounterVec(
|
FilerStoreCounter = prometheus.NewCounterVec(
|
||||||
prometheus.CounterOpts{
|
prometheus.CounterOpts{
|
||||||
Namespace: Namespace,
|
Namespace: Namespace,
|
||||||
@@ -751,6 +761,7 @@ func init() {
|
|||||||
Gather.MustRegister(FilerStoreHistogram)
|
Gather.MustRegister(FilerStoreHistogram)
|
||||||
Gather.MustRegister(FilerSyncOffsetGauge)
|
Gather.MustRegister(FilerSyncOffsetGauge)
|
||||||
Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
|
Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
|
||||||
|
Gather.MustRegister(FilerObjectSizeBytesHistogram)
|
||||||
Gather.MustRegister(collectors.NewGoCollector())
|
Gather.MustRegister(collectors.NewGoCollector())
|
||||||
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user