diff --git a/weed/command/filer_remote_gateway_buckets.go b/weed/command/filer_remote_gateway_buckets.go index 7e22549316..569ef62430 100644 --- a/weed/command/filer_remote_gateway_buckets.go +++ b/weed/command/filer_remote_gateway_buckets.go @@ -214,11 +214,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour } if filer_pb.IsCreate(resp) { if message.NewParentPath == option.bucketsDir { + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } return handleCreateBucket(message.NewEntry) } if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } // Propagate delete markers as deletions on the remote. // Delete markers are zero-content version entries, so they // would be filtered out by the HasData check below. @@ -292,6 +298,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour return nil } if resp.Directory == option.bucketsDir { + if isSystemPath(resp.Directory, message.OldEntry.Name) { + return nil + } return handleDeleteBucket(message.OldEntry) } // Skip deletion of internal version files; individual version @@ -300,6 +309,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name) return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) { + return nil + } bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory) if !ok { return nil @@ -323,6 +335,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour if message.OldEntry.Name == message.NewEntry.Name { return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) || isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } if err := handleCreateBucket(message.NewEntry); err != nil { return err } @@ -339,6 +354,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name) return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + return nil + } oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory) newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath) if oldOk && newOk { diff --git a/weed/command/filer_remote_sync_dir.go b/weed/command/filer_remote_sync_dir.go index da6650f462..d36147191f 100644 --- a/weed/command/filer_remote_sync_dir.go +++ b/weed/command/filer_remote_sync_dir.go @@ -144,6 +144,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) { return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } // Propagate delete markers as deletions on the remote. // Delete markers are zero-content version entries, so they // would be filtered out by the HasData check below. @@ -229,6 +233,10 @@ func processDeleteEvent( glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name) return nil } + if isSystemPath(resp.Directory, message.OldEntry.Name) { + glog.V(2).Infof("skipping delete of system path: %s/%s", resp.Directory, message.OldEntry.Name) + return nil + } if filer.IsMetadataOnlyDelete(message.OldEntry) { glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name) return nil @@ -339,6 +347,10 @@ func processUpdateEvent( glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name) return nil } + if isSystemPath(message.NewParentPath, message.NewEntry.Name) { + glog.V(2).Infof("skipping update of system path: %s/%s", message.NewParentPath, message.NewEntry.Name) + return nil + } oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation) dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation) eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()} @@ -428,7 +440,7 @@ func processUpdateEvent( } glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest)) if err := client.DeleteFile(oldDest); err != nil { - if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) { + if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) || isSystemPath(resp.Directory, message.OldEntry.Name) { return nil } if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) { @@ -881,6 +893,23 @@ func isMultipartUploadDir(dir string) bool { strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/") } +// isSystemPath returns true if the dir and name refer to the internal .system folder +// under /buckets (such as the bucket-owner index at /buckets/.system/owners/...) +// or any entry inside it. These paths are internal to SeaweedFS and must not +// be synced to remote storage. +func isSystemPath(dir string, name string) bool { + fullPath := string(util.NewFullPath(dir, name)) + trimmed := strings.Trim(fullPath, "/") + parts := strings.Split(trimmed, "/") + if len(parts) >= 1 && parts[0] == s3_constants.SystemFolder { + return true + } + if len(parts) >= 2 && parts[1] == s3_constants.SystemFolder { + return true + } + return false +} + // isDeleteMarker returns true if the entry is an S3 delete marker // (a zero-content version entry with ExtDeleteMarkerKey set to "true"). func isDeleteMarker(entry *filer_pb.Entry) bool { diff --git a/weed/command/filer_remote_sync_dir_test.go b/weed/command/filer_remote_sync_dir_test.go index d7fb99ea3f..93c8340b47 100644 --- a/weed/command/filer_remote_sync_dir_test.go +++ b/weed/command/filer_remote_sync_dir_test.go @@ -2016,3 +2016,74 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) { t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry) } } + +func TestIsSystemPath(t *testing.T) { + tests := []struct { + dir string + name string + expected bool + }{ + {dir: "/buckets", name: ".system", expected: true}, + {dir: "/buckets/.system", name: "owners", expected: true}, + {dir: "/buckets/.system/owners", name: "alice", expected: true}, + {dir: "/buckets/.system/owners/alice", name: "mybucket", expected: true}, + {dir: "/", name: ".system", expected: true}, + {dir: "/.system", name: "config", expected: true}, + {dir: "/custom-mount", name: ".system", expected: true}, + {dir: "/custom-mount/.system/owners", name: "alice", expected: true}, + // Bucket contents should not match + {dir: "/buckets/mybucket", name: "file.txt", expected: false}, + {dir: "/buckets/mybucket", name: ".system", expected: false}, + {dir: "/buckets/mybucket/.system", name: "data", expected: false}, + {dir: "/buckets", name: ".system-other", expected: false}, + {dir: "/buckets/.system-other", name: "sub", expected: false}, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s/%s", tt.dir, tt.name), func(t *testing.T) { + got := isSystemPath(tt.dir, tt.name) + if got != tt.expected { + t.Errorf("isSystemPath(%q, %q) = %v, want %v", tt.dir, tt.name, got, tt.expected) + } + }) + } +} + +func TestRemoteSyncSkipsSystemFolder(t *testing.T) { + const mountedDir = "/buckets" + mountLoc := &remote_pb.RemoteStorageLocation{Name: "azure", Bucket: "project-storage", Path: "/"} + + // Test update event on /buckets/.system/owners/... is skipped + updateResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/buckets/.system/owners/structify-api", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + NewParentPath: "/buckets/.system/owners/structify-api", + NewEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + }, + } + remote := &recordingRemote{} + filerClient := &stubFilerClient{} + if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, updateResp); err != nil { + t.Fatalf("processUpdateEvent failed: %v", err) + } + if len(remote.writes) != 0 || len(remote.metadataUpdates) != 0 { + t.Fatalf("expected 0 remote writes or metadata updates for system path, got %d writes and %d metadata updates", + len(remote.writes), len(remote.metadataUpdates)) + } + + // Test delete event on /buckets/.system/owners/... is skipped + deleteResp := &filer_pb.SubscribeMetadataResponse{ + Directory: "/buckets/.system/owners/structify-api", + EventNotification: &filer_pb.EventNotification{ + OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"}, + }, + } + if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleteResp); err != nil { + t.Fatalf("processDeleteEvent failed: %v", err) + } + if len(remote.deletes) != 0 { + t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes)) + } +} + diff --git a/weed/s3api/s3_constants/s3_actions.go b/weed/s3api/s3_constants/s3_actions.go index d90b8cfd32..c920e87cbe 100644 --- a/weed/s3api/s3_constants/s3_actions.go +++ b/weed/s3api/s3_constants/s3_actions.go @@ -29,6 +29,7 @@ const ( SeaweedStorageDestinationHeader = "x-seaweedfs-destination" MultipartUploadsFolder = ".uploads" VersionsFolder = ".versions" + SystemFolder = ".system" FolderMimeType = "httpd/unix-directory" DirectoryMimeType = "application/x-directory" ) diff --git a/weed/s3api/s3api_bucket_owner_index.go b/weed/s3api/s3api_bucket_owner_index.go index b69cbc66a2..a8ccd8ae31 100644 --- a/weed/s3api/s3api_bucket_owner_index.go +++ b/weed/s3api/s3api_bucket_owner_index.go @@ -27,7 +27,7 @@ import ( // indexes pre-existing buckets. The .complete marker is written once the // backfill has finished; until it exists readers keep scanning /buckets. const ( - s3SystemFolder = ".system" + s3SystemFolder = s3_constants.SystemFolder bucketOwnerIndexFolder = "owners" bucketOwnerIndexReadyMarker = ".complete" )