mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-14 18:18:35 +02:00
* feat: prefer same-host worker and local EC replica Admin executor pick ranked idle workers only, so a job whose volume lived on full-03 could still run on full-01 and pull the whole .dat across the LAN. Rank workers whose Hello host matches any replica first (fallback to others if the local worker is full). Detection named Sources[0] / the lowest server id; execute then SyncAndSelectBestReplica and could switch the copy source to another rack. After sync, if a replica on this machine has a complete file count, copy from that URL so affinity and pull stay on one host. * feat: pin scheduler jobs to replica-host workers until load K Reserve prefers the least-loaded replica-host worker and waits while that load is below 5 instead of spilling to a free remote worker. K=0 restores immediate spill. Waterfill uses reservation load so four idle preferred workers each get a job. * feat: park scheduled jobs on per-worker admin queues Assign a detection batch to the least-loaded replica-host worker in one pass, then let ExecutionConcurrency goroutines pop a home-runnable job or steal when that home queue reaches K. Dispatch no longer holds a job in reserve-backoff, so a free worker is not blocked by another home's pin. Tracked jobs stay pending/queued and record the home worker for the admin UI. * fix: rehome queued jobs when the home worker dies A live home still pins below K. A home missing from live executors is stolen immediately onto a free worker so a short orphaned queue does not sit until the dispatch window ends. * feat: wait before stealing a dead home worker queue Orphaned queues are held for 30s so a worker reconnect blink does not dump jobs onto other machines. After the grace, steal without K as before. Zero disables the wait. A home that comes back during grace keeps its queue. * fix: late-bind EC from to worker hostname After replicas are consistent (equal FileCount), rebind encode source to a volume server on the same hostname as worker -address. Any port on that host is local. Diverged FileCount still uses the max replica. * feat: assign homeless jobs to global FIFO Jobs whose replica hostname has no registered worker skip pin and orphan grace. Free workers take own home, then steal at K/orphan, then leftover FIFO. * fix: take homeless FIFO before steal for no-home workers Workers without a replica-host match take leftover first; steal (K/orphan) only when FIFO is empty. Home workers keep own-queue, steal, then leftover. --------- Co-authored-by: Chepurnov Simon <195125934+CHEPURNOV-S-S@users.noreply.github.com>
80 lines
1.7 KiB
Go
80 lines
1.7 KiB
Go
package util
|
|
|
|
import (
|
|
"net"
|
|
"strings"
|
|
)
|
|
|
|
// HostKey is the hostname/IP of an "host:port" address (or the string itself
|
|
// when it has no port). Compared case-insensitively so worker Hello
|
|
// "10.87.0.3:33646" matches a volume replica on "10.87.0.3:8080".
|
|
func HostKey(addr string) string {
|
|
addr = strings.TrimSpace(addr)
|
|
if addr == "" {
|
|
return ""
|
|
}
|
|
host, _, err := net.SplitHostPort(addr)
|
|
if err != nil {
|
|
return strings.ToLower(addr)
|
|
}
|
|
return strings.ToLower(host)
|
|
}
|
|
|
|
// HostKeysFromAddresses returns unique HostKey values, skipping empties.
|
|
func HostKeysFromAddresses(addrs []string) []string {
|
|
seen := make(map[string]struct{}, len(addrs))
|
|
var out []string
|
|
for _, a := range addrs {
|
|
h := HostKey(a)
|
|
if h == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[h]; ok {
|
|
continue
|
|
}
|
|
seen[h] = struct{}{}
|
|
out = append(out, h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// AddressIsLocal reports whether addr's host is this process's machine
|
|
// (loopback or a configured interface address). Used so an EC worker prefers
|
|
// a replica on the same box after replica sync.
|
|
func AddressIsLocal(addr string) bool {
|
|
host := HostKey(addr)
|
|
if host == "" {
|
|
return false
|
|
}
|
|
if host == "localhost" || host == "127.0.0.1" || host == "::1" {
|
|
return true
|
|
}
|
|
ip := net.ParseIP(host)
|
|
if ip != nil && ip.IsLoopback() {
|
|
return true
|
|
}
|
|
ifaces, err := net.InterfaceAddrs()
|
|
if err != nil {
|
|
return false
|
|
}
|
|
for _, ia := range ifaces {
|
|
var got net.IP
|
|
switch v := ia.(type) {
|
|
case *net.IPNet:
|
|
got = v.IP
|
|
case *net.IPAddr:
|
|
got = v.IP
|
|
}
|
|
if got == nil {
|
|
continue
|
|
}
|
|
if strings.EqualFold(got.String(), host) {
|
|
return true
|
|
}
|
|
if ip != nil && got.Equal(ip) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|