mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-20 13:30:46 +02:00
Chris Lu
21054b6c18
Add Guard.IsAdminAuthorized, a fail-closed variant of IsWhiteListed, and use it to gate destructive volume admin RPCs. IsWhiteListed keeps its allow-all-when-empty semantics for HTTP compatibility. For TCP peers with an empty whitelist, off-host callers are rejected but loopback (127.0.0.0/8, ::1) is still trusted. A volume server commonly cohabits with the master/filer on a single host and in integration-test clusters; the loopback exception keeps cluster-internal admin traffic working without -whiteList while still locking out off-host attackers. Non-TCP peers (in-process / bufconn / unix-socket) bypass the host check entirely. When `weed server` runs master+volume+filer in a single process the master dials the volume server in-process and the peer address surfaces as "@", which has no parseable IP. Such a caller shares our OS process and cannot be spoofed by a remote attacker, so we treat it as trusted by construction. The gate also tolerates a nil guard (developmental / embedded path) and only enforces once a guard is wired up. UpdateWhiteList skips entries whose CIDR fails to parse so the IP-iteration path can no longer hit a nil *net.IPNet.
487 lines
15 KiB
Go
487 lines
15 KiB
Go
package weed_server
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/peer"
|
|
"google.golang.org/grpc/status"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/util/version"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/cluster"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
|
)
|
|
|
|
// checkGrpcAdminAuth verifies the gRPC caller is authorized for destructive
|
|
// admin operations by checking the peer address against the guard's whitelist.
|
|
//
|
|
// Only TCP peers go through the host check. Any non-TCP peer (in-process
|
|
// bufconn, unix socket, etc.) is treated as trusted because such callers
|
|
// share the same OS process or host filesystem as the volume server and
|
|
// cannot be spoofed by a remote attacker. This matters when `weed server`
|
|
// runs master+volume+filer in a single process and the master invokes
|
|
// AllocateVolume through an in-process gRPC dial: the peer address surfaces
|
|
// as "@" rather than 127.0.0.1, which has no parseable IP.
|
|
//
|
|
// The whitelist check uses Guard.IsAdminAuthorized, which is fail-closed: a
|
|
// volume server with a configured guard but empty whitelist still rejects
|
|
// destructive admin RPCs from off-host TCP peers (loopback TCP callers are
|
|
// trusted, see IsAdminAuthorized). Operators must set -whiteList on the CLI
|
|
// or guard.white_list in security.toml to allow remote admin peers. Failed
|
|
// attempts are logged so misconfigured callers and probe attempts are
|
|
// visible.
|
|
func (vs *VolumeServer) checkGrpcAdminAuth(ctx context.Context) error {
|
|
if vs.guard == nil {
|
|
// No guard wired up at all — this is a developmental / embedded path
|
|
// (e.g. unit tests). Skip the gate; the real protection lives in
|
|
// IsAdminAuthorized, which only takes effect once a guard exists.
|
|
glog.V(1).Infof("gRPC admin auth: no guard configured; set -whiteList or guard.white_list in security.toml to enforce")
|
|
return nil
|
|
}
|
|
pr, ok := peer.FromContext(ctx)
|
|
if !ok {
|
|
// Real gRPC connections always populate peer info; if we don't know
|
|
// who the caller is, deny.
|
|
glog.V(0).Infof("gRPC admin auth failed: no peer info")
|
|
return status.Error(codes.PermissionDenied, "no peer info")
|
|
}
|
|
tcpAddr, isTCP := pr.Addr.(*net.TCPAddr)
|
|
if !isTCP {
|
|
// In-process / bufconn / unix-socket peer: same OS process as us, so
|
|
// trusted by construction. A remote attacker cannot reach this path.
|
|
return nil
|
|
}
|
|
host := tcpAddr.IP.String()
|
|
if !vs.guard.IsAdminAuthorized(host) {
|
|
glog.V(0).Infof("gRPC admin auth failed: %s is not authorized (remote: %s)", host, pr.Addr.String())
|
|
return status.Errorf(codes.PermissionDenied, "not authorized: %s", host)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (vs *VolumeServer) DeleteCollection(ctx context.Context, req *volume_server_pb.DeleteCollectionRequest) (*volume_server_pb.DeleteCollectionResponse, error) {
|
|
|
|
resp := &volume_server_pb.DeleteCollectionResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
err := vs.store.DeleteCollection(req.Collection)
|
|
|
|
if err != nil {
|
|
glog.Errorf("delete collection %s: %v", req.Collection, err)
|
|
} else {
|
|
glog.V(2).Infof("delete collection %v", req)
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) AllocateVolume(ctx context.Context, req *volume_server_pb.AllocateVolumeRequest) (*volume_server_pb.AllocateVolumeResponse, error) {
|
|
resp := &volume_server_pb.AllocateVolumeResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
if err := vs.CheckMaintenanceMode(); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
err := vs.store.AddVolume(
|
|
needle.VolumeId(req.VolumeId),
|
|
req.Collection,
|
|
vs.needleMapKind,
|
|
req.Replication,
|
|
req.Ttl,
|
|
req.Preallocate,
|
|
needle.Version(req.Version),
|
|
req.MemoryMapMaxSizeMb,
|
|
types.ToDiskType(req.DiskType),
|
|
vs.ldbTimout,
|
|
)
|
|
|
|
if err != nil {
|
|
glog.Errorf("assign volume %v: %v", req, err)
|
|
} else {
|
|
glog.V(2).Infof("assign volume %v", req)
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeMount(ctx context.Context, req *volume_server_pb.VolumeMountRequest) (*volume_server_pb.VolumeMountResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeMountResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
err := vs.store.MountVolume(needle.VolumeId(req.VolumeId))
|
|
|
|
if err != nil {
|
|
glog.Errorf("volume mount %v: %v", req, err)
|
|
} else {
|
|
glog.V(2).Infof("volume mount %v", req)
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeUnmount(ctx context.Context, req *volume_server_pb.VolumeUnmountRequest) (*volume_server_pb.VolumeUnmountResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeUnmountResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId))
|
|
|
|
if err != nil {
|
|
glog.Errorf("volume unmount %v: %v", req, err)
|
|
} else {
|
|
glog.V(2).Infof("volume unmount %v", req)
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
|
|
resp := &volume_server_pb.VolumeDeleteResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
if err := vs.CheckMaintenanceMode(); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId), req.OnlyEmpty, req.KeepRemoteData)
|
|
|
|
if err != nil {
|
|
glog.Errorf("volume delete %v: %v", req, err)
|
|
} else {
|
|
glog.V(2).Infof("volume delete %v", req)
|
|
}
|
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeConfigure(ctx context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
|
|
resp := &volume_server_pb.VolumeConfigureResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
if err := vs.CheckMaintenanceMode(); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
// check replication format
|
|
if _, err := super_block.NewReplicaPlacementFromString(req.Replication); err != nil {
|
|
resp.Error = fmt.Sprintf("volume configure replication %v: %v", req, err)
|
|
return resp, nil
|
|
}
|
|
|
|
// unmount
|
|
if err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId)); err != nil {
|
|
glog.Errorf("volume configure unmount %v: %v", req, err)
|
|
resp.Error = fmt.Sprintf("volume configure unmount %v: %v", req, err)
|
|
return resp, nil
|
|
}
|
|
|
|
// modify the volume info file
|
|
if err := vs.store.ConfigureVolume(needle.VolumeId(req.VolumeId), req.Replication); err != nil {
|
|
glog.Errorf("volume configure %v: %v", req, err)
|
|
resp.Error = fmt.Sprintf("volume configure %v: %v", req, err)
|
|
// Try to re-mount to restore the volume state
|
|
if mountErr := vs.store.MountVolume(needle.VolumeId(req.VolumeId)); mountErr != nil {
|
|
glog.Errorf("volume configure failed to restore mount %v: %v", req, mountErr)
|
|
resp.Error += fmt.Sprintf(". Also failed to restore mount: %v", mountErr)
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
// mount
|
|
if err := vs.store.MountVolume(needle.VolumeId(req.VolumeId)); err != nil {
|
|
glog.Errorf("volume configure mount %v: %v", req, err)
|
|
resp.Error = fmt.Sprintf("volume configure mount %v: %v", req, err)
|
|
return resp, nil
|
|
}
|
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) makeVolumeReadonly(ctx context.Context, v *storage.Volume, persist bool) error {
|
|
if err := vs.CheckMaintenanceMode(); err != nil {
|
|
return err
|
|
}
|
|
|
|
// step 1: stop master from redirecting traffic here
|
|
if err := vs.notifyMasterVolumeReadonly(ctx, v, true); err != nil {
|
|
return err
|
|
}
|
|
|
|
// rare case 1.5: it will be unlucky if heartbeat happened between step 1 and 2.
|
|
|
|
// step 2: mark local volume as readonly
|
|
if err := vs.store.MarkVolumeReadonly(v.Id, persist); err != nil {
|
|
glog.Errorf("mark volume %d readonly: %v", v.Id, err)
|
|
return err
|
|
} else {
|
|
glog.V(2).Infof("volume %d marked readonly", v.Id)
|
|
}
|
|
|
|
// step 3: tell master from redirecting traffic here again, to prevent rare case 1.5
|
|
if err := vs.notifyMasterVolumeReadonly(ctx, v, true); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (vs *VolumeServer) makeVolumeWritable(ctx context.Context, v *storage.Volume) error {
|
|
if err := vs.CheckMaintenanceMode(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := vs.store.MarkVolumeWritable(v.Id); err != nil {
|
|
glog.Errorf("mark volume %d writable: %v", v.Id, err)
|
|
return err
|
|
} else {
|
|
glog.V(2).Infof("volume %d marked writable", v.Id)
|
|
}
|
|
|
|
// enable master to redirect traffic here
|
|
if err := vs.notifyMasterVolumeReadonly(ctx, v, false); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (vs *VolumeServer) notifyMasterVolumeReadonly(ctx context.Context, v *storage.Volume, isReadOnly bool) error {
|
|
if grpcErr := pb.WithMasterClient(false, vs.GetMaster(ctx), vs.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
|
_, err := client.VolumeMarkReadonly(context.Background(), &master_pb.VolumeMarkReadonlyRequest{
|
|
Ip: vs.store.Ip,
|
|
Port: uint32(vs.store.Port),
|
|
VolumeId: uint32(v.Id),
|
|
Collection: v.Collection,
|
|
ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
|
|
Ttl: v.Ttl.ToUint32(),
|
|
DiskType: string(v.DiskType()),
|
|
IsReadonly: isReadOnly,
|
|
})
|
|
if err != nil {
|
|
return fmt.Errorf("set volume %d to read only on master: %v", v.Id, err)
|
|
}
|
|
return nil
|
|
}); grpcErr != nil {
|
|
glog.V(0).Infof("connect to %s: %v", vs.GetMaster(context.Background()), grpcErr)
|
|
return fmt.Errorf("grpc VolumeMarkReadonly with master %s: %v", vs.GetMaster(context.Background()), grpcErr)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
|
|
resp := &volume_server_pb.VolumeMarkReadonlyResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
|
|
if v == nil {
|
|
return resp, fmt.Errorf("volume %d not found", req.VolumeId)
|
|
}
|
|
|
|
if err := vs.makeVolumeReadonly(ctx, v, req.GetPersist()); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
|
|
resp := &volume_server_pb.VolumeMarkWritableResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
|
|
if v == nil {
|
|
return resp, fmt.Errorf("volume %d not found", req.VolumeId)
|
|
}
|
|
|
|
if err := vs.makeVolumeWritable(ctx, v); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeStatus(ctx context.Context, req *volume_server_pb.VolumeStatusRequest) (*volume_server_pb.VolumeStatusResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeStatusResponse{}
|
|
|
|
v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
|
|
if v == nil {
|
|
return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
|
|
}
|
|
if v.DataBackend == nil {
|
|
return nil, fmt.Errorf("volume %d data backend not found", req.VolumeId)
|
|
}
|
|
|
|
volumeSize, _, _ := v.DataBackend.GetStat()
|
|
resp.IsReadOnly = v.IsReadOnly()
|
|
resp.VolumeSize = uint64(volumeSize)
|
|
resp.FileCount = v.FileCount()
|
|
resp.FileDeletedCount = v.DeletedCount()
|
|
|
|
return resp, nil
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_server_pb.VolumeServerStatusRequest) (*volume_server_pb.VolumeServerStatusResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeServerStatusResponse{
|
|
State: vs.store.State.Proto(),
|
|
MemoryStatus: stats.MemStat(),
|
|
Version: version.Version(),
|
|
DataCenter: vs.dataCenter,
|
|
Rack: vs.rack,
|
|
}
|
|
|
|
for _, loc := range vs.store.Locations {
|
|
if dir, e := filepath.Abs(loc.Directory); e == nil {
|
|
resp.DiskStatuses = append(resp.DiskStatuses, stats.NewDiskStatus(dir))
|
|
}
|
|
}
|
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeServerLeave(ctx context.Context, req *volume_server_pb.VolumeServerLeaveRequest) (*volume_server_pb.VolumeServerLeaveResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeServerLeaveResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
vs.StopHeartbeat()
|
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) VolumeNeedleStatus(ctx context.Context, req *volume_server_pb.VolumeNeedleStatusRequest) (*volume_server_pb.VolumeNeedleStatusResponse, error) {
|
|
|
|
resp := &volume_server_pb.VolumeNeedleStatusResponse{}
|
|
|
|
if err := vs.checkGrpcAdminAuth(ctx); err != nil {
|
|
return resp, err
|
|
}
|
|
|
|
volumeId := needle.VolumeId(req.VolumeId)
|
|
|
|
n := &needle.Needle{
|
|
Id: types.NeedleId(req.NeedleId),
|
|
}
|
|
|
|
var count int
|
|
var err error
|
|
hasVolume := vs.store.HasVolume(volumeId)
|
|
if !hasVolume {
|
|
_, hasEcVolume := vs.store.FindEcVolume(volumeId)
|
|
if !hasEcVolume {
|
|
return nil, fmt.Errorf("volume not found %d", req.VolumeId)
|
|
}
|
|
count, err = vs.store.ReadEcShardNeedle(volumeId, n, nil)
|
|
} else {
|
|
count, err = vs.store.ReadVolumeNeedle(volumeId, n, nil, nil)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if count < 0 {
|
|
return nil, fmt.Errorf("needle not found %d", n.Id)
|
|
}
|
|
|
|
resp.NeedleId = uint64(n.Id)
|
|
resp.Cookie = uint32(n.Cookie)
|
|
resp.Size = uint32(n.Size)
|
|
resp.LastModified = n.LastModified
|
|
resp.Crc = n.Checksum.Value()
|
|
if n.HasTtl() {
|
|
resp.Ttl = n.Ttl.String()
|
|
}
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
func (vs *VolumeServer) Ping(ctx context.Context, req *volume_server_pb.PingRequest) (resp *volume_server_pb.PingResponse, pingErr error) {
|
|
resp = &volume_server_pb.PingResponse{
|
|
StartTimeNs: time.Now().UnixNano(),
|
|
}
|
|
if req.TargetType == cluster.FilerType {
|
|
pingErr = pb.WithFilerClient(false, 0, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
|
pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
|
|
if pingResp != nil {
|
|
resp.RemoteTimeNs = pingResp.StartTimeNs
|
|
}
|
|
return err
|
|
})
|
|
}
|
|
if req.TargetType == cluster.VolumeServerType {
|
|
pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
|
|
pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
|
|
if pingResp != nil {
|
|
resp.RemoteTimeNs = pingResp.StartTimeNs
|
|
}
|
|
return err
|
|
})
|
|
}
|
|
if req.TargetType == cluster.MasterType {
|
|
pingErr = pb.WithMasterClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
|
|
pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
|
|
if pingResp != nil {
|
|
resp.RemoteTimeNs = pingResp.StartTimeNs
|
|
}
|
|
return err
|
|
})
|
|
}
|
|
if pingErr != nil {
|
|
pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
|
|
}
|
|
resp.StopTimeNs = time.Now().UnixNano()
|
|
return
|
|
}
|