weed filer.remote.sync: ignore .system directory under buckets (#11966) (#11969)

weed/command/filer_remote_gateway_buckets.go

@@ -214,11 +214,17 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 		}
 		if filer_pb.IsCreate(resp) {
 			if message.NewParentPath == option.bucketsDir {
+				if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+					return nil
+				}
 				return handleCreateBucket(message.NewEntry)
 			}
 			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			// Propagate delete markers as deletions on the remote.
 			// Delete markers are zero-content version entries, so they
 			// would be filtered out by the HasData check below.
@@ -292,6 +298,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				return nil
 			}
 			if resp.Directory == option.bucketsDir {
+				if isSystemPath(resp.Directory, message.OldEntry.Name) {
+					return nil
+				}
 				return handleDeleteBucket(message.OldEntry)
 			}
 			// Skip deletion of internal version files; individual version
@@ -300,6 +309,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
 				return nil
 			}
+			if isSystemPath(resp.Directory, message.OldEntry.Name) {
+				return nil
+			}
 			bucket, remoteStorageMountLocation, remoteStorage, ok := option.detectBucketInfo(resp.Directory)
 			if !ok {
 				return nil
@@ -323,6 +335,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 					if message.OldEntry.Name == message.NewEntry.Name {
 						return nil
 					}
+					if isSystemPath(resp.Directory, message.OldEntry.Name) || isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+						return nil
+					}
 					if err := handleCreateBucket(message.NewEntry); err != nil {
 						return err
 					}
@@ -339,6 +354,9 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
 				glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				return nil
+			}
 			oldBucket, oldRemoteStorageMountLocation, oldRemoteStorage, oldOk := option.detectBucketInfo(resp.Directory)
 			newBucket, newRemoteStorageMountLocation, newRemoteStorage, newOk := option.detectBucketInfo(message.NewParentPath)
 			if oldOk && newOk {

weed/command/filer_remote_sync_dir.go

@@ -144,6 +144,10 @@ func (option *RemoteSyncOptions) makeEventProcessor(remoteStorage *remote_pb.Rem
 			if isMultipartUploadFile(message.NewParentPath, message.NewEntry.Name) {
 				return nil
 			}
+			if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+				glog.V(2).Infof("skipping create of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
+				return nil
+			}
 			// Propagate delete markers as deletions on the remote.
 			// Delete markers are zero-content version entries, so they
 			// would be filtered out by the HasData check below.
@@ -229,6 +233,10 @@ func processDeleteEvent(
 		glog.V(2).Infof("skipping delete of internal version path: %s/%s", resp.Directory, message.OldEntry.Name)
 		return nil
 	}
+	if isSystemPath(resp.Directory, message.OldEntry.Name) {
+		glog.V(2).Infof("skipping delete of system path: %s/%s", resp.Directory, message.OldEntry.Name)
+		return nil
+	}
 	if filer.IsMetadataOnlyDelete(message.OldEntry) {
 		glog.V(2).Infof("skipping remote delete of metadata-only delete: %s/%s", resp.Directory, message.OldEntry.Name)
 		return nil
@@ -339,6 +347,10 @@ func processUpdateEvent(
 		glog.V(2).Infof("skipping update of internal version path: %s/%s", message.NewParentPath, message.NewEntry.Name)
 		return nil
 	}
+	if isSystemPath(message.NewParentPath, message.NewEntry.Name) {
+		glog.V(2).Infof("skipping update of system path: %s/%s", message.NewParentPath, message.NewEntry.Name)
+		return nil
+	}
 	oldDest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(resp.Directory, message.OldEntry.Name), remoteStorageMountLocation)
 	dest := toRemoteStorageLocation(util.FullPath(mountedDir), util.NewFullPath(message.NewParentPath, message.NewEntry.Name), remoteStorageMountLocation)
 	eventSource := renameSource{path: util.NewFullPath(resp.Directory, message.OldEntry.Name), loc: oldDest, stamp: message.OldEntry.GetRemoteEntry()}
@@ -428,7 +440,7 @@ func processUpdateEvent(
 		}
 		glog.V(0).Infof("delete %s", remote_storage.FormatLocation(oldDest))
 		if err := client.DeleteFile(oldDest); err != nil {
-			if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) {
+			if isMultipartUploadFile(resp.Directory, message.OldEntry.Name) || isSystemPath(resp.Directory, message.OldEntry.Name) {
 				return nil
 			}
 			if !errors.Is(err, remote_storage.ErrRemoteObjectNotFound) {
@@ -881,6 +893,23 @@ func isMultipartUploadDir(dir string) bool {
 		strings.Contains(dir, "/"+s3_constants.MultipartUploadsFolder+"/")
 }
 
+// isSystemPath returns true if the dir and name refer to the internal .system folder
+// under /buckets (such as the bucket-owner index at /buckets/.system/owners/...)
+// or any entry inside it. These paths are internal to SeaweedFS and must not
+// be synced to remote storage.
+func isSystemPath(dir string, name string) bool {
+	fullPath := string(util.NewFullPath(dir, name))
+	trimmed := strings.Trim(fullPath, "/")
+	parts := strings.Split(trimmed, "/")
+	if len(parts) >= 1 && parts[0] == s3_constants.SystemFolder {
+		return true
+	}
+	if len(parts) >= 2 && parts[1] == s3_constants.SystemFolder {
+		return true
+	}
+	return false
+}
+
 // isDeleteMarker returns true if the entry is an S3 delete marker
 // (a zero-content version entry with ExtDeleteMarkerKey set to "true").
 func isDeleteMarker(entry *filer_pb.Entry) bool {

weed/command/filer_remote_sync_dir_test.go

@@ -2016,3 +2016,74 @@ func TestMetadataUpdateKeepsTheStoredStampOnNilResult(t *testing.T) {
 		t.Fatalf("stamped RemoteEntry = %+v, want the entry's own stamp kept (mtime 100)", filerClient.updated.Entry.RemoteEntry)
 	}
 }
+
+func TestIsSystemPath(t *testing.T) {
+	tests := []struct {
+		dir      string
+		name     string
+		expected bool
+	}{
+		{dir: "/buckets", name: ".system", expected: true},
+		{dir: "/buckets/.system", name: "owners", expected: true},
+		{dir: "/buckets/.system/owners", name: "alice", expected: true},
+		{dir: "/buckets/.system/owners/alice", name: "mybucket", expected: true},
+		{dir: "/", name: ".system", expected: true},
+		{dir: "/.system", name: "config", expected: true},
+		{dir: "/custom-mount", name: ".system", expected: true},
+		{dir: "/custom-mount/.system/owners", name: "alice", expected: true},
+		// Bucket contents should not match
+		{dir: "/buckets/mybucket", name: "file.txt", expected: false},
+		{dir: "/buckets/mybucket", name: ".system", expected: false},
+		{dir: "/buckets/mybucket/.system", name: "data", expected: false},
+		{dir: "/buckets", name: ".system-other", expected: false},
+		{dir: "/buckets/.system-other", name: "sub", expected: false},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s/%s", tt.dir, tt.name), func(t *testing.T) {
+			got := isSystemPath(tt.dir, tt.name)
+			if got != tt.expected {
+				t.Errorf("isSystemPath(%q, %q) = %v, want %v", tt.dir, tt.name, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestRemoteSyncSkipsSystemFolder(t *testing.T) {
+	const mountedDir = "/buckets"
+	mountLoc := &remote_pb.RemoteStorageLocation{Name: "azure", Bucket: "project-storage", Path: "/"}
+
+	// Test update event on /buckets/.system/owners/... is skipped
+	updateResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/.system/owners/structify-api",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry:      &filer_pb.Entry{Name: "structify-copy-canary"},
+			NewParentPath: "/buckets/.system/owners/structify-api",
+			NewEntry:      &filer_pb.Entry{Name: "structify-copy-canary"},
+		},
+	}
+	remote := &recordingRemote{}
+	filerClient := &stubFilerClient{}
+	if err := processUpdateEvent(filerClient, filerClient, "", remote, mountedDir, mountLoc, updateResp); err != nil {
+		t.Fatalf("processUpdateEvent failed: %v", err)
+	}
+	if len(remote.writes) != 0 || len(remote.metadataUpdates) != 0 {
+		t.Fatalf("expected 0 remote writes or metadata updates for system path, got %d writes and %d metadata updates",
+			len(remote.writes), len(remote.metadataUpdates))
+	}
+
+	// Test delete event on /buckets/.system/owners/... is skipped
+	deleteResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/buckets/.system/owners/structify-api",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{Name: "structify-copy-canary"},
+		},
+	}
+	if err := processDeleteEvent(filerClient, remote, mountedDir, mountLoc, deleteResp); err != nil {
+		t.Fatalf("processDeleteEvent failed: %v", err)
+	}
+	if len(remote.deletes) != 0 {
+		t.Fatalf("expected 0 remote deletes for system path, got %d deletes", len(remote.deletes))
+	}
+}
+

weed/s3api/s3_constants/s3_actions.go

@@ -29,6 +29,7 @@ const (
 	SeaweedStorageDestinationHeader = "x-seaweedfs-destination"
 	MultipartUploadsFolder          = ".uploads"
 	VersionsFolder                  = ".versions"
+	SystemFolder                    = ".system"
 	FolderMimeType                  = "httpd/unix-directory"
 	DirectoryMimeType               = "application/x-directory"
 )

weed/s3api/s3api_bucket_owner_index.go

@@ -27,7 +27,7 @@ import (
 // indexes pre-existing buckets. The .complete marker is written once the
 // backfill has finished; until it exists readers keep scanning /buckets.
 const (
-	s3SystemFolder              = ".system"
+	s3SystemFolder              = s3_constants.SystemFolder
 	bucketOwnerIndexFolder      = "owners"
 	bucketOwnerIndexReadyMarker = ".complete"
 )