volume: resolve the disk IO slow-latency threshold per disk (#10976)

* volume: resolve the disk IO slow-latency threshold per disk

volume.toml keys [volume.disk.io.slow.latency] by disk type, but the
threshold was chosen once per server by switching on the raw -disk flag.
-disk is comma-separated, one entry per -dir, so a multi-disk server
matched no case and silently took the hdd threshold.

Carry the table on DiskIOProbeConfig and resolve it in CheckDiskSpace
from the location's own DiskType. A type with no entry keeps falling
back to the hdd threshold.

* volume: run the disk IO probe on multi-directory volume servers

The probe was disabled whenever more than one -dir was configured,
because a single server-wide slow-latency threshold could not describe
disks of different types. The threshold is per disk now, and the rest of
the probe already is: diskRegistry is keyed by directory, each
DiskLocation runs its own CheckDiskSpace, and Store consults
isDiskUnavailable per location.

* volume: reject duplicate -dir entries

Nothing deduplicated -dir, so the same directory listed twice produced two
DiskLocations that each loaded every volume in it, appending to the same .dat
under two independent locks. Compare directory identity with os.SameFile
rather than the path, so a symlink or bind mount aliasing an earlier entry is
rejected as well.

* volume: cover the per-disk slow-latency handoff

SlowLatencyFor has a test, but nothing asserted that CheckDiskSpace feeds it
the location's own disk type. Probe through a seam so the resolved threshold
is observable, and check hdd, ssd, nvme, the empty type, and an unlisted tag.

weed/command/scaffold/volume.toml

@@ -21,6 +21,8 @@ percent = 20
 [volume.disk.io.error]
 percent = 10
 
+# slow threshold per disk type, matched against each -dir's -disk tag.
+# a disk whose type is not listed here uses the hdd threshold.
 [volume.disk.io.slow.latency]
 hdd = "500ms"
 ssd = "100ms"

weed/command/volume.go

@@ -293,11 +293,29 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	// Set multiple folders and each folder's max volume count limit'
 	v.folders = strings.Split(volumeFolders, ",")
+	// Two locations on one directory would load every volume twice, appending to
+	// the same .dat under two independent locks. Compare identity rather than the
+	// path, so a symlink or bind mount aliasing an earlier -dir is caught too.
+	type seenFolder struct {
+		path string
+		info os.FileInfo
+	}
+	var seenFolders []seenFolder
 	for i, folder := range v.folders {
 		v.folders[i] = util.ResolvePath(folder)
 		if err := util.TestFolderWritable(v.folders[i]); err != nil {
 			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", v.folders[i], err)
 		}
+		folderInfo, err := os.Stat(v.folders[i])
+		if err != nil {
+			glog.Fatalf("Check Data Folder(-dir) %s : %s", v.folders[i], err)
+		}
+		for _, seen := range seenFolders {
+			if os.SameFile(seen.info, folderInfo) {
+				glog.Fatalf("Data Folder(-dir) %s and %s are the same directory", seen.path, v.folders[i])
+			}
+		}
+		seenFolders = append(seenFolders, seenFolder{v.folders[i], folderInfo})
 	}
 
 	// set max
@@ -396,24 +414,18 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	// Determine volume server ID: if not specified, use ip:port
 	volumeServerId := util.GetVolumeServerId(*v.id, *v.ip, *v.port)
-	var slowLatency time.Duration
 
-	switch *v.diskType {
-	case "hdd":
-		slowLatency = *v.diskHDDIOSlowLatency
-	case "ssd":
-		slowLatency = *v.diskSSDIOSlowLatency
-	case "nvme":
-		slowLatency = *v.diskNVMEIOSlowLatency
-	default:
-		slowLatency = *v.diskHDDIOSlowLatency
-	}
 	diskProbeConfig := stats_collect.DiskIOProbeConfig{
 		Enabled:  *v.diskIOProbe,
 		Timeout:  *v.diskIOTimeout,
 		Interval: *v.diskIOInterval,
 
-		SlowLatency: slowLatency,
+		SlowLatency: *v.diskHDDIOSlowLatency,
+		SlowLatencyByDiskType: map[string]time.Duration{
+			types.HddType:  *v.diskHDDIOSlowLatency,
+			types.SsdType:  *v.diskSSDIOSlowLatency,
+			types.NvmeType: *v.diskNVMEIOSlowLatency,
+		},
 
 		Window:     *v.diskIOWindow,
 		MinSamples: *v.diskIOMinSamples,
@@ -425,10 +437,6 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 		RecoveryCoef: *v.diskRecoveryCoef,
 	}
-	if diskProbeConfig.Enabled && len(v.folders) > 1 {
-		glog.Warningf("disk IO probe is disabled for multiple volume directories: %v", v.folders)
-		diskProbeConfig.Enabled = false
-	}
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.portGrpc, *v.publicUrl, volumeServerId,
 		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes, folderTags,