mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-19 02:15:02 +01:00
* log_buffer: let a subscriber skip wake-ups for unwanted partition keys Every append woke every registered subscriber. SetSubscriberWakeFilter lets a reader sleep through appends whose partition key it would filter out; it still reads them on its next poll. Flushes and keyless appends wake everyone. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer_pb: decide subscription matches from an event's directories MetadataDirectoryMayMatchSubscription rules out events whose directory no child path of can match the subscription, and ScanMetadataEventDirectories reads an event's directory and new parent path without allocating. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: wake subscribers of a move's destination directory Appends are keyed by the source directory, so a filtered subscriber of the destination would otherwise only see the move on its next poll. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: wake metadata subscribers only for directories they can match Every metadata event woke and was scanned by every subscriber, so filer CPU grew with the number of subscribers watching other paths. Readers now register a directory wake filter, and the per-entry prefilter rules entries out from their directories before the skeleton scan. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * filer: test that moves into a subscribed prefix pass the prefilter Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
479 lines
15 KiB
Go
479 lines
15 KiB
Go
package filer_pb
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/viant/ptrie"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/protobuf/proto"
|
|
)
|
|
|
|
const cutoffTimeNewEmptyDir = 3
|
|
|
|
func (entry *Entry) IsInRemoteOnly() bool {
|
|
return len(entry.GetChunks()) == 0 && entry.RemoteEntry != nil && entry.RemoteEntry.RemoteSize > 0
|
|
}
|
|
|
|
func (entry *Entry) IsDirectoryKeyObject() bool {
|
|
// Also true for a file promoted to a directory by a child write, which keeps its
|
|
// chunks/content, or its remote entry when the file was tiered to remote storage.
|
|
return entry.IsDirectory &&
|
|
((entry.Attributes != nil && entry.Attributes.Mime != "") || len(entry.GetChunks()) > 0 || len(entry.GetContent()) > 0 || entry.IsInRemoteOnly() || entry.IsPrefixObject())
|
|
}
|
|
|
|
// IsPrefixObject reports whether the directory entry also holds the object named by its
|
|
// path without a trailing slash, which an empty object leaves no other trace of.
|
|
func (entry *Entry) IsPrefixObject() bool {
|
|
if entry == nil || !entry.IsDirectory {
|
|
return false
|
|
}
|
|
_, marked := entry.Extended[s3_constants.SeaweedFSPrefixObject]
|
|
return marked
|
|
}
|
|
|
|
// MarkPrefixObject turns a file entry into the directory entry that stores it, so keys
|
|
// nested under its key can be created beside it.
|
|
func (entry *Entry) MarkPrefixObject() {
|
|
entry.IsDirectory = true
|
|
if entry.Attributes != nil {
|
|
entry.Attributes.FileMode |= uint32(os.ModeDir) | 0111
|
|
}
|
|
if entry.Extended == nil {
|
|
entry.Extended = make(map[string][]byte)
|
|
}
|
|
entry.Extended[s3_constants.SeaweedFSPrefixObject] = []byte("true")
|
|
}
|
|
|
|
func (entry *Entry) GetExpiryTime() (expiryTime int64) {
|
|
// For S3 objects with lifecycle expiration, use Mtime (modification time)
|
|
// For regular TTL entries, use Crtime (creation time) for backward compatibility
|
|
if entry.Extended != nil {
|
|
if _, hasS3Expiry := entry.Extended[s3_constants.SeaweedFSExpiresS3]; hasS3Expiry {
|
|
// S3 lifecycle expiration: base TTL on modification time
|
|
expiryTime = entry.Attributes.Mtime
|
|
if expiryTime == 0 {
|
|
expiryTime = entry.Attributes.Crtime
|
|
}
|
|
expiryTime += int64(entry.Attributes.TtlSec)
|
|
return expiryTime
|
|
}
|
|
}
|
|
|
|
// Regular TTL expiration: base on creation time only
|
|
expiryTime = entry.Attributes.Crtime + int64(entry.Attributes.TtlSec)
|
|
return expiryTime
|
|
}
|
|
|
|
func (entry *Entry) IsExpired() bool {
|
|
return entry != nil && entry.Attributes != nil && entry.Attributes.TtlSec > 0 &&
|
|
time.Now().Unix() >= entry.GetExpiryTime()
|
|
}
|
|
|
|
func (entry *Entry) FileMode() (fileMode os.FileMode) {
|
|
if entry != nil && entry.Attributes != nil {
|
|
fileMode = os.FileMode(entry.Attributes.FileMode)
|
|
}
|
|
return
|
|
}
|
|
|
|
func (entry *Entry) IsOlderDir() bool {
|
|
return entry.IsDirectory && entry.Attributes != nil && entry.Attributes.Mime == "" && entry.Attributes.GetCrtime() <= time.Now().Unix()-cutoffTimeNewEmptyDir
|
|
}
|
|
|
|
func ToFileIdObject(fileIdStr string) (*FileId, error) {
|
|
t, err := needle.ParseFileIdFromString(fileIdStr)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &FileId{
|
|
VolumeId: uint32(t.VolumeId),
|
|
Cookie: uint32(t.Cookie),
|
|
FileKey: uint64(t.Key),
|
|
}, nil
|
|
|
|
}
|
|
|
|
func (fid *FileId) toFileIdString() string {
|
|
return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
|
|
}
|
|
|
|
func (c *FileChunk) GetFileIdString() string {
|
|
if c.FileId != "" {
|
|
return c.FileId
|
|
}
|
|
if c.Fid != nil {
|
|
c.FileId = c.Fid.toFileIdString()
|
|
return c.FileId
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// FileIdsString lists chunks by file id only, for logging that must not print
|
|
// per-chunk fields like CipherKey.
|
|
func FileIdsString(chunks []*FileChunk) []string {
|
|
fileIds := make([]string, 0, len(chunks))
|
|
for _, chunk := range chunks {
|
|
fileIds = append(fileIds, chunk.GetFileIdString())
|
|
}
|
|
return fileIds
|
|
}
|
|
|
|
func BeforeEntrySerialization(chunks []*FileChunk) {
|
|
|
|
for _, chunk := range chunks {
|
|
|
|
if chunk.FileId != "" {
|
|
if fid, err := ToFileIdObject(chunk.FileId); err == nil {
|
|
chunk.Fid = fid
|
|
chunk.FileId = ""
|
|
}
|
|
}
|
|
|
|
if chunk.SourceFileId != "" {
|
|
if fid, err := ToFileIdObject(chunk.SourceFileId); err == nil {
|
|
chunk.SourceFid = fid
|
|
chunk.SourceFileId = ""
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func AfterEntryDeserialization(chunks []*FileChunk) {
|
|
|
|
for _, chunk := range chunks {
|
|
|
|
if chunk.Fid != nil && chunk.FileId == "" {
|
|
chunk.FileId = chunk.Fid.toFileIdString()
|
|
}
|
|
|
|
if chunk.SourceFid != nil && chunk.SourceFileId == "" {
|
|
chunk.SourceFileId = chunk.SourceFid.toFileIdString()
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func CreateEntry(ctx context.Context, client SeaweedFilerClient, request *CreateEntryRequest) error {
|
|
_, err := CreateEntryWithResponse(ctx, client, request)
|
|
return err
|
|
}
|
|
|
|
func CreateEntryWithResponse(ctx context.Context, client SeaweedFilerClient, request *CreateEntryRequest) (*CreateEntryResponse, error) {
|
|
resp, err := client.CreateEntry(ctx, request)
|
|
if err != nil {
|
|
glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v", request.Directory, request.Entry.Name, request.OExcl, err)
|
|
return nil, fmt.Errorf("CreateEntry: %w", err)
|
|
}
|
|
if resp.ErrorCode != FilerError_OK {
|
|
glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v (code %v)", request.Directory, request.Entry.Name, request.OExcl, resp.Error, resp.ErrorCode)
|
|
if sentinel := FilerErrorToSentinel(resp.ErrorCode); sentinel != nil {
|
|
return nil, fmt.Errorf("CreateEntry %s/%s: %w", request.Directory, request.Entry.Name, sentinel)
|
|
}
|
|
return nil, fmt.Errorf("CreateEntry: %w", errors.New(resp.Error))
|
|
}
|
|
if resp.Error != "" {
|
|
glog.V(1).InfofCtx(ctx, "create entry %s/%s %v: %v", request.Directory, request.Entry.Name, request.OExcl, resp.Error)
|
|
return nil, fmt.Errorf("CreateEntry: %w", errors.New(resp.Error))
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
// SnapshotExtended clones extended attributes for use as an UpdateEntry
|
|
// ExpectedExtended precondition. Listed keys are asserted even when absent,
|
|
// so a concurrent add still fails the precondition.
|
|
func SnapshotExtended(extended map[string][]byte, keys ...string) map[string][]byte {
|
|
expected := make(map[string][]byte, len(extended)+len(keys))
|
|
for k, v := range extended {
|
|
expected[k] = v
|
|
}
|
|
for _, k := range keys {
|
|
if _, ok := expected[k]; !ok {
|
|
expected[k] = nil
|
|
}
|
|
}
|
|
return expected
|
|
}
|
|
|
|
func UpdateEntry(ctx context.Context, client SeaweedFilerClient, request *UpdateEntryRequest) error {
|
|
_, err := UpdateEntryWithResponse(ctx, client, request)
|
|
return err
|
|
}
|
|
|
|
func UpdateEntryWithResponse(ctx context.Context, client SeaweedFilerClient, request *UpdateEntryRequest) (*UpdateEntryResponse, error) {
|
|
resp, err := client.UpdateEntry(ctx, request)
|
|
if err != nil {
|
|
glog.V(1).InfofCtx(ctx, "update entry %s/%s :%v", request.Directory, request.Entry.Name, err)
|
|
return nil, fmt.Errorf("UpdateEntry: %w", err)
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
func LookupEntry(ctx context.Context, client SeaweedFilerClient, request *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) {
|
|
resp, err := client.LookupDirectoryEntry(ctx, request)
|
|
if err != nil {
|
|
if err == ErrNotFound {
|
|
return nil, ErrNotFound
|
|
}
|
|
// The filer converts its own miss to codes.NotFound; a raw sentinel
|
|
// crossing gRPC arrives as codes.Unknown carrying only its text. The
|
|
// same text on any other status is a failed read, not a miss.
|
|
if st, ok := util.ServerStatus(err); ok {
|
|
if st.Code() == codes.NotFound ||
|
|
(st.Code() == codes.Unknown && strings.Contains(st.Message(), ErrNotFound.Error())) {
|
|
return nil, ErrNotFound
|
|
}
|
|
} else if strings.Contains(err.Error(), ErrNotFound.Error()) {
|
|
return nil, ErrNotFound
|
|
}
|
|
glog.V(3).InfofCtx(ctx, "read %s/%v: %v", request.Directory, request.Name, err)
|
|
return nil, fmt.Errorf("LookupEntry1: %w", err)
|
|
}
|
|
if resp.Entry == nil {
|
|
return nil, ErrNotFound
|
|
}
|
|
return resp, nil
|
|
}
|
|
|
|
var ErrNotFound = errors.New("filer: no entry is found in filer store")
|
|
|
|
// Sentinel errors for filer entry operations.
|
|
// These are set by the filer and reconstructed from FilerError codes after
|
|
// crossing the gRPC boundary, so consumers can use errors.Is() instead of
|
|
// parsing error strings.
|
|
var (
|
|
ErrEntryNameTooLong = errors.New("entry name too long")
|
|
ErrParentIsFile = errors.New("parent path is a file")
|
|
ErrExistingIsDirectory = errors.New("existing entry is a directory")
|
|
ErrExistingIsFile = errors.New("existing entry is a file")
|
|
ErrEntryAlreadyExists = errors.New("entry already exists")
|
|
ErrReadOnly = errors.New("read only")
|
|
ErrPreconditionFailed = errors.New("precondition failed")
|
|
)
|
|
|
|
// FilerErrorToSentinel maps a proto FilerError code to its sentinel error.
|
|
// Returns nil for OK or unknown codes.
|
|
func FilerErrorToSentinel(code FilerError) error {
|
|
switch code {
|
|
case FilerError_ENTRY_NAME_TOO_LONG:
|
|
return ErrEntryNameTooLong
|
|
case FilerError_PARENT_IS_FILE:
|
|
return ErrParentIsFile
|
|
case FilerError_EXISTING_IS_DIRECTORY:
|
|
return ErrExistingIsDirectory
|
|
case FilerError_EXISTING_IS_FILE:
|
|
return ErrExistingIsFile
|
|
case FilerError_ENTRY_ALREADY_EXISTS:
|
|
return ErrEntryAlreadyExists
|
|
case FilerError_READ_ONLY:
|
|
return ErrReadOnly
|
|
case FilerError_PRECONDITION_FAILED:
|
|
return ErrPreconditionFailed
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
// AssignVolumeResponseError returns the failure resp reports, or nil. The error
|
|
// keeps the filer's text and, when error_code is set, unwraps to its sentinel so
|
|
// callers can match it with errors.Is.
|
|
func AssignVolumeResponseError(resp *AssignVolumeResponse) error {
|
|
if resp.Error == "" && resp.ErrorCode == FilerError_OK {
|
|
return nil
|
|
}
|
|
return &codedError{msg: resp.Error, sentinel: FilerErrorToSentinel(resp.ErrorCode)}
|
|
}
|
|
|
|
// codedError is a filer error message paired with the sentinel of the code
|
|
// sent alongside it (nil for OK or an unknown code).
|
|
type codedError struct {
|
|
msg string
|
|
sentinel error
|
|
}
|
|
|
|
func (e *codedError) Error() string { return e.msg }
|
|
func (e *codedError) Unwrap() error { return e.sentinel }
|
|
|
|
func IsEmpty(event *SubscribeMetadataResponse) bool {
|
|
return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry == nil
|
|
}
|
|
|
|
func IsCreate(event *SubscribeMetadataResponse) bool {
|
|
return event.EventNotification.NewEntry != nil && event.EventNotification.OldEntry == nil
|
|
}
|
|
|
|
func IsUpdate(event *SubscribeMetadataResponse) bool {
|
|
return event.EventNotification.NewEntry != nil &&
|
|
event.EventNotification.OldEntry != nil &&
|
|
event.Directory == event.EventNotification.NewParentPath &&
|
|
event.EventNotification.NewEntry.Name == event.EventNotification.OldEntry.Name
|
|
}
|
|
|
|
func IsDelete(event *SubscribeMetadataResponse) bool {
|
|
return event.EventNotification.NewEntry == nil && event.EventNotification.OldEntry != nil
|
|
}
|
|
|
|
func IsRename(event *SubscribeMetadataResponse) bool {
|
|
return event.EventNotification.NewEntry != nil &&
|
|
event.EventNotification.OldEntry != nil &&
|
|
(event.Directory != event.EventNotification.NewParentPath ||
|
|
event.EventNotification.NewEntry.Name != event.EventNotification.OldEntry.Name)
|
|
}
|
|
|
|
func MetadataEventSourceDirectory(event *SubscribeMetadataResponse) string {
|
|
if event == nil {
|
|
return ""
|
|
}
|
|
return event.Directory
|
|
}
|
|
|
|
func MetadataEventTargetDirectory(event *SubscribeMetadataResponse) string {
|
|
if event == nil {
|
|
return ""
|
|
}
|
|
if event.EventNotification != nil && event.EventNotification.NewParentPath != "" {
|
|
return event.EventNotification.NewParentPath
|
|
}
|
|
return event.Directory
|
|
}
|
|
|
|
func metadataEventSourceEntryName(event *SubscribeMetadataResponse) string {
|
|
if event == nil || event.EventNotification == nil {
|
|
return ""
|
|
}
|
|
if event.EventNotification.OldEntry != nil {
|
|
return event.EventNotification.OldEntry.Name
|
|
}
|
|
if event.EventNotification.NewEntry != nil {
|
|
return event.EventNotification.NewEntry.Name
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func metadataEventTargetEntryName(event *SubscribeMetadataResponse) string {
|
|
if event == nil || event.EventNotification == nil {
|
|
return ""
|
|
}
|
|
if event.EventNotification.NewEntry != nil {
|
|
return event.EventNotification.NewEntry.Name
|
|
}
|
|
if event.EventNotification.OldEntry != nil {
|
|
return event.EventNotification.OldEntry.Name
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func MetadataEventSourceFullPath(event *SubscribeMetadataResponse) string {
|
|
return util.Join(MetadataEventSourceDirectory(event), metadataEventSourceEntryName(event))
|
|
}
|
|
|
|
func MetadataEventTargetFullPath(event *SubscribeMetadataResponse) string {
|
|
return util.Join(MetadataEventTargetDirectory(event), metadataEventTargetEntryName(event))
|
|
}
|
|
|
|
func MetadataEventTouchesDirectory(event *SubscribeMetadataResponse, dir string) bool {
|
|
if MetadataEventSourceDirectory(event) == dir {
|
|
return true
|
|
}
|
|
return event != nil &&
|
|
event.EventNotification != nil &&
|
|
event.EventNotification.NewEntry != nil &&
|
|
MetadataEventTargetDirectory(event) == dir
|
|
}
|
|
|
|
func MetadataEventMatchesSubscription(event *SubscribeMetadataResponse, pathPrefix string, pathPrefixes []string, directories []string) bool {
|
|
if event == nil {
|
|
return false
|
|
}
|
|
|
|
if metadataEventMatchesPath(MetadataEventSourceFullPath(event), MetadataEventSourceDirectory(event), pathPrefix, pathPrefixes, directories) {
|
|
return true
|
|
}
|
|
|
|
return event.EventNotification != nil &&
|
|
event.EventNotification.NewEntry != nil &&
|
|
metadataEventMatchesPath(MetadataEventTargetFullPath(event), MetadataEventTargetDirectory(event), pathPrefix, pathPrefixes, directories)
|
|
}
|
|
|
|
func metadataEventMatchesPath(fullPath, dirPath, pathPrefix string, pathPrefixes []string, directories []string) bool {
|
|
if hasPrefixIn(fullPath, pathPrefixes) {
|
|
return true
|
|
}
|
|
if matchByDirectory(dirPath, directories) {
|
|
return true
|
|
}
|
|
return strings.HasPrefix(fullPath, pathPrefix)
|
|
}
|
|
|
|
// MetadataDirectoryMayMatchSubscription reports whether an event in dir could
|
|
// match the subscription, before the event is decoded. It errs toward true.
|
|
func MetadataDirectoryMayMatchSubscription(dir []byte, pathPrefix string, pathPrefixes []string, directories []string) bool {
|
|
if pathPrefix == "" || len(dir) == 0 || directoryMayHavePrefix(dir, pathPrefix) {
|
|
return true
|
|
}
|
|
for _, p := range pathPrefixes {
|
|
if directoryMayHavePrefix(dir, p) {
|
|
return true
|
|
}
|
|
}
|
|
for _, d := range directories {
|
|
if string(dir) == d {
|
|
return true
|
|
}
|
|
}
|
|
// matching cleans the joined path, so only a clean dir can be ruled out
|
|
for i := 0; i+1 < len(dir); i++ {
|
|
if dir[i] == '/' && (dir[i+1] == '/' || dir[i+1] == '.') {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// directoryMayHavePrefix reports whether some path dir/<name> can start with
|
|
// prefix: dir+"/" and prefix agree on their common length.
|
|
func directoryMayHavePrefix(dir []byte, prefix string) bool {
|
|
n := min(len(dir), len(prefix))
|
|
if string(dir[:n]) != prefix[:n] {
|
|
return false
|
|
}
|
|
if len(prefix) <= len(dir) || dir[len(dir)-1] == '/' {
|
|
return true
|
|
}
|
|
return prefix[len(dir)] == '/'
|
|
}
|
|
|
|
func hasPrefixIn(text string, prefixes []string) bool {
|
|
for _, p := range prefixes {
|
|
if strings.HasPrefix(text, p) {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func matchByDirectory(dirPath string, directories []string) bool {
|
|
for _, dir := range directories {
|
|
if dirPath == dir {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
var _ = ptrie.KeyProvider(&FilerConf_PathConf{})
|
|
|
|
func (fp *FilerConf_PathConf) Key() interface{} {
|
|
key, _ := proto.Marshal(fp)
|
|
return string(key)
|
|
}
|