seaweedfs/weed/server/volume_grpc_admin.go

package weed_server

import (
	"context"
	"errors"
	"fmt"
	"net"
	"path/filepath"
	"strings"
	"time"

	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/peer"
	"google.golang.org/grpc/status"

	"github.com/seaweedfs/seaweedfs/weed/util/version"

	"github.com/seaweedfs/seaweedfs/weed/storage"

	"github.com/seaweedfs/seaweedfs/weed/cluster"
	"github.com/seaweedfs/seaweedfs/weed/glog"
	"github.com/seaweedfs/seaweedfs/weed/pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
	"github.com/seaweedfs/seaweedfs/weed/stats"
	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
	"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
	"github.com/seaweedfs/seaweedfs/weed/storage/types"
	"github.com/seaweedfs/seaweedfs/weed/storage/volume_info"
)

// checkGrpcAdminAuth verifies the gRPC caller is authorized for destructive
// admin operations by checking the peer address against the guard's whitelist.
//
// IP extraction prefers a typed *net.TCPAddr where available, falling back to
// SplitHostPort on the string form, then to the raw string. The fallback
// chain matters because in-process/passthrough connections used in tests
// surface as unparseable strings like "@"; with an empty whitelist the
// allow-all branch in IsWhiteListed accepts them, with a whitelist they're
// denied as expected.
//
// Failed authorization attempts are logged so an operator running with a
// configured whitelist can spot misconfigured callers and probe attempts.
func (vs *VolumeServer) checkGrpcAdminAuth(ctx context.Context) error {
	if vs.guard == nil {
		return nil
	}
	pr, ok := peer.FromContext(ctx)
	if !ok {
		// Real gRPC connections always populate peer info; if we don't know
		// who the caller is, deny.
		glog.V(0).Infof("gRPC admin auth failed: no peer info")
		return status.Error(codes.PermissionDenied, "no peer info")
	}
	addr := pr.Addr.String()
	var host string
	if tcpAddr, ok := pr.Addr.(*net.TCPAddr); ok {
		host = tcpAddr.IP.String()
	} else if h, _, splitErr := net.SplitHostPort(addr); splitErr == nil {
		host = h
	} else {
		host = addr
	}
	if !vs.guard.IsWhiteListed(host) {
		glog.V(0).Infof("gRPC admin auth failed: %s is not whitelisted (remote: %s)", host, addr)
		return status.Errorf(codes.PermissionDenied, "not authorized: %s", host)
	}
	return nil
}

func (vs *VolumeServer) DeleteCollection(ctx context.Context, req *volume_server_pb.DeleteCollectionRequest) (*volume_server_pb.DeleteCollectionResponse, error) {

	resp := &volume_server_pb.DeleteCollectionResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	err := vs.store.DeleteCollection(req.Collection)

	if err != nil {
		glog.Errorf("delete collection %s: %v", req.Collection, err)
		return resp, volumeStatusError(fmt.Errorf("delete collection %s: %w", req.Collection, err))
	}
	glog.V(2).Infof("delete collection %v", req)

	return resp, err

}

func (vs *VolumeServer) AllocateVolume(ctx context.Context, req *volume_server_pb.AllocateVolumeRequest) (*volume_server_pb.AllocateVolumeResponse, error) {
	resp := &volume_server_pb.AllocateVolumeResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	if err := vs.CheckMaintenanceMode(); err != nil {
		return resp, err
	}

	// Same window as VolumeMount: allocation registers the volume, and a
	// copy in flight clears or rewrites the files it would open.
	guard := vs.volumeCopyGuard(req.VolumeId)
	guard.Lock()
	defer guard.Unlock()
	if guard.copying {
		return resp, status.Errorf(codes.Unavailable, "volume %d is being copied", req.VolumeId)
	}

	err := vs.store.AddVolume(
		needle.VolumeId(req.VolumeId),
		req.Collection,
		vs.needleMapKind,
		req.Replication,
		req.Ttl,
		req.Preallocate,
		needle.Version(req.Version),
		req.MemoryMapMaxSizeMb,
		types.ToDiskType(req.DiskType),
		vs.ldbTimout,
	)

	if err != nil {
		glog.Errorf("assign volume %v: %v", req, err)
	} else {
		glog.V(2).Infof("assign volume %v", req)
	}

	return resp, err

}

func (vs *VolumeServer) VolumeMount(ctx context.Context, req *volume_server_pb.VolumeMountRequest) (*volume_server_pb.VolumeMountResponse, error) {

	resp := &volume_server_pb.VolumeMountResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	// A copy claims the volume, then waits on the source. A mount in that
	// window loads files the copy is about to clear or overwrite, so the
	// check and the registration share the copy's guard: a mount either
	// registers before the copy's recheck (and makes it abort) or sees the
	// claim and refuses.
	guard := vs.volumeCopyGuard(req.VolumeId)
	guard.Lock()
	if guard.copying {
		guard.Unlock()
		return resp, status.Errorf(codes.Unavailable, "volume %d is being copied", req.VolumeId)
	}
	err := vs.store.MountVolumeClaimed(needle.VolumeId(req.VolumeId), req.Collection)
	guard.Unlock()

	if err != nil {
		glog.Errorf("volume mount %v: %v", req, err)
	} else {
		glog.V(2).Infof("volume mount %v", req)
	}

	return resp, err

}

func (vs *VolumeServer) VolumeUnmount(ctx context.Context, req *volume_server_pb.VolumeUnmountRequest) (*volume_server_pb.VolumeUnmountResponse, error) {

	resp := &volume_server_pb.VolumeUnmountResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	// Same window as VolumeMount: the copy announces its replica before the
	// claim drops, and an unmount inside that window unregisters a volume the
	// master was just told is here.
	guard := vs.volumeCopyGuard(req.VolumeId)
	guard.Lock()
	defer guard.Unlock()
	if guard.copying {
		return resp, status.Errorf(codes.Unavailable, "volume %d is being copied", req.VolumeId)
	}

	err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId))

	if err != nil {
		glog.Errorf("volume unmount %v: %v", req, err)
	} else {
		glog.V(2).Infof("volume unmount %v", req)
	}

	return resp, err

}

func (vs *VolumeServer) VolumeConsolidateIndex(ctx context.Context, req *volume_server_pb.VolumeConsolidateIndexRequest) (*volume_server_pb.VolumeConsolidateIndexResponse, error) {

	resp := &volume_server_pb.VolumeConsolidateIndexResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	if err := vs.CheckMaintenanceMode(); err != nil {
		return resp, err
	}

	err := vs.store.ConsolidateVolumeIndex(needle.VolumeId(req.VolumeId))

	if err != nil {
		glog.Errorf("volume consolidate index %v: %v", req, err)
	} else {
		glog.V(2).Infof("volume consolidate index %v", req)
	}

	return resp, err

}

// VolumeDelete is allowed in maintenance mode: it removes data from the server
// rather than adding any, and evacuating a server in maintenance mode ends each
// move by deleting the source copy (issue #11066).
func (vs *VolumeServer) VolumeDelete(ctx context.Context, req *volume_server_pb.VolumeDeleteRequest) (*volume_server_pb.VolumeDeleteResponse, error) {
	resp := &volume_server_pb.VolumeDeleteResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	// Same window as VolumeMount: a delete inside the copy's post-mount
	// window removes a replica the master was just told is here.
	guard := vs.volumeCopyGuard(req.VolumeId)
	guard.Lock()
	defer guard.Unlock()
	if guard.copying {
		return resp, status.Errorf(codes.Unavailable, "volume %d is being copied", req.VolumeId)
	}
	// Match the Rust server, which drops a cancelled RPC before it runs.
	if err := ctx.Err(); err != nil {
		glog.Warningf("volume delete %v not started: %v", req, err)
		return resp, status.FromContextError(err).Err()
	}

	err := vs.store.DeleteVolume(needle.VolumeId(req.VolumeId), req.OnlyEmpty, req.OnlyGarbage, req.KeepRemoteData)

	if err != nil {
		glog.Errorf("volume delete %v: %v", req, err)
		return resp, volumeStatusError(err)
	} else {
		// V(0) so destructive RPCs are always traceable.
		glog.Infof("volume delete %v", req)
	}

	return resp, err

}

// volumeStatusError keeps the store's message so callers matching on
// "not found" or "volume not empty" keep working, and adds the status code so
// new callers do not have to.
func volumeStatusError(err error) error {
	if errors.Is(err, storage.ErrVolumeNotFound) {
		return status.Error(codes.NotFound, err.Error())
	}
	if errors.Is(err, storage.ErrVolumeNotEmpty) {
		return status.Error(codes.FailedPrecondition, err.Error())
	}
	if errors.Is(err, storage.ErrInsufficientSpace) {
		return status.Error(codes.ResourceExhausted, err.Error())
	}
	return err
}

func (vs *VolumeServer) VolumeConfigure(ctx context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
	resp := &volume_server_pb.VolumeConfigureResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	if err := vs.CheckMaintenanceMode(); err != nil {
		return resp, err
	}

	// check replication format
	if _, err := super_block.NewReplicaPlacementFromString(req.Replication); err != nil {
		resp.Error = fmt.Sprintf("volume configure replication %v: %v", req, err)
		return resp, nil
	}

	// Same window as VolumeMount, and wider: configure unmounts, rewrites
	// the .vif, and remounts, so the whole sequence shares the copy's guard
	// rather than only the entry check.
	guard := vs.volumeCopyGuard(req.VolumeId)
	guard.Lock()
	defer guard.Unlock()
	if guard.copying {
		return resp, status.Errorf(codes.Unavailable, "volume %d is being copied", req.VolumeId)
	}

	// unmount
	if err := vs.store.UnmountVolume(needle.VolumeId(req.VolumeId)); err != nil {
		glog.Errorf("volume configure unmount %v: %v", req, err)
		resp.Error = fmt.Sprintf("volume configure unmount %v: %v", req, err)
		return resp, nil
	}

	// modify the volume info file
	if err := vs.store.ConfigureVolume(needle.VolumeId(req.VolumeId), req.Replication); err != nil {
		glog.Errorf("volume configure %v: %v", req, err)
		resp.Error = fmt.Sprintf("volume configure %v: %v", req, err)
		// Try to re-mount to restore the volume state
		if mountErr := vs.store.MountVolumeClaimed(needle.VolumeId(req.VolumeId), nil); mountErr != nil {
			glog.Errorf("volume configure failed to restore mount %v: %v", req, mountErr)
			resp.Error += fmt.Sprintf(". Also failed to restore mount: %v", mountErr)
		}
		return resp, nil
	}

	// mount
	if err := vs.store.MountVolumeClaimed(needle.VolumeId(req.VolumeId), nil); err != nil {
		glog.Errorf("volume configure mount %v: %v", req, err)
		resp.Error = fmt.Sprintf("volume configure mount %v: %v", req, err)
		return resp, nil
	}

	return resp, nil

}

// makeVolumeReadonly is not gated on maintenance mode: marking a volume readonly
// only restricts a server that is already meant to be read-only, and it is the
// first step of moving a volume off a server under evacuation (issue #11066).
func (vs *VolumeServer) makeVolumeReadonly(ctx context.Context, v *storage.Volume, canDelete bool, persist bool) error {
	// step 1: stop master from redirecting traffic here
	if err := vs.notifyMasterVolumeReadonly(ctx, v, true); err != nil {
		return err
	}

	// rare case 1.5: it will be unlucky if heartbeat happened between step 1 and 2.

	// step 2: mark local volume as readonly
	var persistErr error
	if err := vs.store.MarkVolumeReadonly(v.Id, canDelete, persist); err != nil {
		var ndErr *volume_info.NotCrashDurableError
		if !errors.As(err, &ndErr) {
			glog.Errorf("mark volume %d readonly: %v", v.Id, err)
			return err
		}
		// Post-rename durability failure: the .vif already holds the new
		// mode. Continue with step 3 so the master reflects the change,
		// then propagate the durability warning.
		glog.Warningf("mark volume %d readonly: %v", v.Id, err)
		persistErr = err
	} else {
		glog.V(2).Infof("volume %d marked readonly", v.Id)
	}

	// step 3: tell master from redirecting traffic here again, to prevent rare case 1.5
	if err := vs.notifyMasterVolumeReadonly(ctx, v, true); err != nil {
		return err
	}

	return persistErr
}

func (vs *VolumeServer) makeVolumeWritable(ctx context.Context, v *storage.Volume) error {
	if err := vs.CheckMaintenanceMode(); err != nil {
		return err
	}

	var persistErr error
	if err := vs.store.MarkVolumeWritable(v.Id); err != nil {
		var ndErr *volume_info.NotCrashDurableError
		if !errors.As(err, &ndErr) {
			glog.Errorf("mark volume %d writable: %v", v.Id, err)
			return err
		}
		// Post-rename durability failure: the .vif already holds the new
		// mode. Continue notifying the master so traffic is redirected,
		// then propagate the durability warning.
		glog.Warningf("mark volume %d writable: %v", v.Id, err)
		persistErr = err
	} else {
		glog.V(2).Infof("volume %d marked writable", v.Id)
	}

	// enable master to redirect traffic here
	if err := vs.notifyMasterVolumeReadonly(ctx, v, false); err != nil {
		return err
	}

	return persistErr
}

func isNotLeaderErr(err error) bool {
	return err != nil && strings.Contains(err.Error(), "Not current leader")
}

func (vs *VolumeServer) notifyMasterVolumeReadonly(ctx context.Context, v *storage.Volume, isReadOnly bool) error {
	master := vs.GetMaster(ctx)
	err := vs.volumeMarkReadonlyOnMaster(ctx, master, v, isReadOnly)
	if err != nil && isNotLeaderErr(err) {
		leader, lookupErr := vs.lookupRaftLeaderMaster(ctx)
		if lookupErr != nil {
			return fmt.Errorf("heartbeat master %s rejected mark-readonly and leader lookup failed: %w", master, lookupErr)
		}
		master = leader
		err = vs.volumeMarkReadonlyOnMaster(ctx, master, v, isReadOnly)
	}
	return err
}

func (vs *VolumeServer) volumeMarkReadonlyOnMaster(ctx context.Context, master pb.ServerAddress, v *storage.Volume, isReadOnly bool) error {
	if grpcErr := pb.WithMasterClient(ctx, false, master, vs.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
		_, err := client.VolumeMarkReadonly(ctx, &master_pb.VolumeMarkReadonlyRequest{
			Ip:               vs.store.Ip,
			Port:             uint32(vs.store.Port),
			VolumeId:         uint32(v.Id),
			Collection:       v.Collection,
			ReplicaPlacement: uint32(v.ReplicaPlacement.Byte()),
			Ttl:              v.Ttl.ToUint32(),
			DiskType:         string(v.DiskType()),
			IsReadonly:       isReadOnly,
		})
		if err != nil {
			return fmt.Errorf("set volume %d to read only on master: %v", v.Id, err)
		}
		return nil
	}); grpcErr != nil {
		glog.V(0).Infof("connect to %s: %v", master, grpcErr)
		return fmt.Errorf("grpc VolumeMarkReadonly with master %s: %v", master, grpcErr)
	}
	return nil
}

func (vs *VolumeServer) VolumeMarkReadonly(ctx context.Context, req *volume_server_pb.VolumeMarkReadonlyRequest) (*volume_server_pb.VolumeMarkReadonlyResponse, error) {
	resp := &volume_server_pb.VolumeMarkReadonlyResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return resp, fmt.Errorf("volume %d not found", req.VolumeId)
	}

	if err := vs.makeVolumeReadonly(ctx, v, req.GetCanDelete(), req.GetPersist()); err != nil {
		return resp, err
	}

	return resp, nil
}

func (vs *VolumeServer) VolumeMarkWritable(ctx context.Context, req *volume_server_pb.VolumeMarkWritableRequest) (*volume_server_pb.VolumeMarkWritableResponse, error) {
	resp := &volume_server_pb.VolumeMarkWritableResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return resp, fmt.Errorf("volume %d not found", req.VolumeId)
	}

	if err := vs.makeVolumeWritable(ctx, v); err != nil {
		return resp, err
	}

	return resp, nil
}

func (vs *VolumeServer) VolumeStatus(ctx context.Context, req *volume_server_pb.VolumeStatusRequest) (*volume_server_pb.VolumeStatusResponse, error) {

	resp := &volume_server_pb.VolumeStatusResponse{}

	v := vs.store.GetVolume(needle.VolumeId(req.VolumeId))
	if v == nil {
		return nil, fmt.Errorf("not found volume id %d", req.VolumeId)
	}
	if v.DataBackend == nil {
		return nil, fmt.Errorf("volume %d data backend not found", req.VolumeId)
	}

	volumeSize, _, _ := v.DataBackend.GetStat()
	resp.IsReadOnly = v.IsReadOnly()
	resp.VolumeSize = uint64(volumeSize)
	resp.FileCount = v.FileCount()
	resp.FileDeletedCount = v.DeletedCount()

	return resp, nil
}

func (vs *VolumeServer) VolumeServerStatus(ctx context.Context, req *volume_server_pb.VolumeServerStatusRequest) (*volume_server_pb.VolumeServerStatusResponse, error) {

	resp := &volume_server_pb.VolumeServerStatusResponse{
		State:        vs.store.State.Proto(),
		MemoryStatus: stats.MemStat(),
		Version:      version.Version(),
		DataCenter:   vs.dataCenter,
		Rack:         vs.rack,
	}

	for _, loc := range vs.store.Locations {
		if dir, e := filepath.Abs(loc.Directory); e == nil {
			resp.DiskStatuses = append(resp.DiskStatuses, stats.NewDiskStatus(dir))
		}
	}

	return resp, nil

}

func (vs *VolumeServer) VolumeServerLeave(ctx context.Context, req *volume_server_pb.VolumeServerLeaveRequest) (*volume_server_pb.VolumeServerLeaveResponse, error) {

	resp := &volume_server_pb.VolumeServerLeaveResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	vs.StopHeartbeat()

	return resp, nil

}

func (vs *VolumeServer) VolumeNeedleStatus(ctx context.Context, req *volume_server_pb.VolumeNeedleStatusRequest) (*volume_server_pb.VolumeNeedleStatusResponse, error) {

	resp := &volume_server_pb.VolumeNeedleStatusResponse{}

	if err := vs.checkGrpcAdminAuth(ctx); err != nil {
		return resp, err
	}

	volumeId := needle.VolumeId(req.VolumeId)

	n := &needle.Needle{
		Id: types.NeedleId(req.NeedleId),
	}

	var count int
	var err error
	hasVolume := vs.store.HasVolume(volumeId)
	if !hasVolume {
		_, hasEcVolume := vs.store.FindEcVolume(volumeId)
		if !hasEcVolume {
			return nil, fmt.Errorf("volume not found %d", req.VolumeId)
		}
		count, err = vs.store.ReadEcShardNeedle(volumeId, n, nil)
	} else {
		count, err = vs.store.ReadVolumeNeedle(volumeId, n, nil, nil)
	}
	if err != nil {
		if errors.Is(err, storage.ErrorNotFound) || errors.Is(err, erasure_coding.NotFoundError) {
			return nil, status.Errorf(codes.NotFound, "needle not found %d", n.Id)
		}
		return nil, err
	}
	if count < 0 {
		return nil, status.Errorf(codes.NotFound, "needle not found %d", n.Id)
	}

	resp.NeedleId = uint64(n.Id)
	resp.Cookie = uint32(n.Cookie)
	resp.Size = uint32(n.Size)
	resp.LastModified = n.LastModified
	resp.Crc = n.Checksum.Value()
	if n.HasTtl() {
		resp.Ttl = n.Ttl.String()
	}
	return resp, nil

}

// isKnownPingTarget reports whether target is a master this volume server
// already knows about. Volume servers do not maintain a peer-volume or
// peer-filer list, so Ping is scoped to the masters they heartbeat with.
// The current-master read is taken under a lock to avoid racing with the
// heartbeat goroutine that rewrites it on leader changes, and the seed
// list is consulted via a pre-built set so the check stays O(1).
func (vs *VolumeServer) isKnownPingTarget(target string, targetType string) bool {
	if targetType != cluster.MasterType {
		return false
	}
	addr := pb.ServerAddress(target)
	key := addr.ToHttpAddress()
	if key == "" {
		return false
	}
	if current := vs.getCurrentMaster(); current != "" && current.ToHttpAddress() == key {
		return true
	}
	_, ok := vs.seedMasterSet[key]
	return ok
}

func (vs *VolumeServer) Ping(ctx context.Context, req *volume_server_pb.PingRequest) (resp *volume_server_pb.PingResponse, pingErr error) {
	resp = &volume_server_pb.PingResponse{
		StartTimeNs: time.Now().UnixNano(),
	}
	// Empty target is a self-liveness probe and stays unauthenticated.
	if req.Target != "" && !vs.isKnownPingTarget(req.Target, req.TargetType) {
		resp.StopTimeNs = time.Now().UnixNano()
		return resp, status.Errorf(codes.InvalidArgument, "unknown ping target %s of type %s", req.Target, req.TargetType)
	}
	if req.TargetType == cluster.FilerType {
		pingErr = pb.WithFilerClient(false, 0, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			pingResp, err := client.Ping(ctx, &filer_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if req.TargetType == cluster.VolumeServerType {
		pingErr = pb.WithVolumeServerClient(false, pb.ServerAddress(req.Target), vs.grpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
			pingResp, err := client.Ping(ctx, &volume_server_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if req.TargetType == cluster.MasterType {
		pingErr = pb.WithMasterClient(context.Background(), false, pb.ServerAddress(req.Target), vs.grpcDialOption, false, func(client master_pb.SeaweedClient) error {
			pingResp, err := client.Ping(ctx, &master_pb.PingRequest{})
			if pingResp != nil {
				resp.RemoteTimeNs = pingResp.StartTimeNs
			}
			return err
		})
	}
	if pingErr != nil {
		pingErr = fmt.Errorf("ping %s %s: %v", req.TargetType, req.Target, pingErr)
	}
	resp.StopTimeNs = time.Now().UnixNano()
	return
}