filer: delete a copied rename's old object only after the commit

Review follow-ups:

- The old key of a copied remote-only entry, and the old directory of a
  rename that copied one, are now deleted once the rename has committed.
  A rename that fails after the copy leaves the source entry and its
  object in place, plus an extra object at the new key.
- Every part of a multipart copy is conditioned on the ETag the copy
  started from, so an object overwritten mid-copy fails the copy
  instead of mixing versions.
- Both copy paths keep the source's server-side encryption settings.
- filer.remote.sync no longer deletes a key that a remote-only entry
  holds by the time the delete event is processed: a file deleted and
  then replaced by a remote-only rename has its content only there.

weed/server/filer_grpc_server_rename.go

@@ -150,9 +150,18 @@ type metadataEvent struct {
 	oldEntry     *filer.Entry
 	newEntry     *filer.Entry
 	deleteChunks bool
+	// the old entry's remote object is deleted once the rename has committed:
+	// it, or an object under it, was copied to the new key, and a rename that
+	// fails must not lose it
+	deleteOldRemote bool
 }
 
 func (event metadataEvent) notify(f *filer.Filer, ctx context.Context, signatures []int32) {
+	if event.deleteOldRemote {
+		if err := f.DeleteFromRemote(ctx, event.oldEntry); err != nil {
+			glog.WarningfCtx(ctx, "delete remote object of renamed %s: %v", event.oldEntry.FullPath, err)
+		}
+	}
 	f.NotifyUpdateEvent(ctx, event.oldEntry, event.newEntry, event.deleteChunks, false, signatures)
 }
 
@@ -336,19 +345,33 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seawee
 		})
 	}
 	*metadataEvents = append(*metadataEvents, metadataEvent{
-		oldEntry: sourceEntry,
-		newEntry: newEntry,
+		oldEntry:        sourceEntry,
+		newEntry:        newEntry,
+		deleteOldRemote: copiedRemote != nil,
 	})
+	selfEvent := len(*metadataEvents) - 1
 
 	if moveFolderSubEntries != nil {
 		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
 			return moveChildrenErr
 		}
+		// removing the old directory on the remote removes every object under
+		// it, so it waits for the commit too once a child was copied
+		for _, event := range (*metadataEvents)[selfEvent+1:] {
+			if event.deleteOldRemote {
+				(*metadataEvents)[selfEvent].deleteOldRemote = true
+				break
+			}
+		}
 	}
 
 	// delete old entry
 	ctx = context.WithValue(ctx, "OP", "MV")
-	deleteErr := fs.filer.DeleteEntryMetaAndData(filer.WithSuppressedMetadataEvents(ctx), oldPath, false, false, false, false, signatures, 0)
+	deleteCtx := filer.WithSuppressedMetadataEvents(ctx)
+	if (*metadataEvents)[selfEvent].deleteOldRemote {
+		deleteCtx = filer.WithKeepRemoteObject(deleteCtx)
+	}
+	deleteErr := fs.filer.DeleteEntryMetaAndData(deleteCtx, oldPath, false, false, false, false, signatures, 0)
 	if deleteErr != nil {
 		return deleteErr
 	}

weed/server/filer_grpc_server_rename_remote_test.go

@@ -226,18 +226,26 @@ func TestRenameEmptyRemoteEntryDoesNotCopy(t *testing.T) {
 	assert.Equal(t, []string{"delete origin/src/empty", "delete origin/m/empty"}, client.recorded())
 }
 
-func TestRenameRemoteOnlyEntryFailingAfterTheCopyKeepsTheContent(t *testing.T) {
-	client := copyingRenameRemoteClient{&renameRemoteClient{}}
-	server, store := newRenameRemoteTestServer(t, client)
-	store.entries["/buckets/b/src/a.jpg"] = remoteOnlyRenameEntry("/buckets/b/src/a.jpg", 101)
-	store.commitErr = errors.New("commit failed")
+func TestRenameRemoteOnlyEntryFailingAfterTheCopyKeepsTheSource(t *testing.T) {
+	for _, src := range []util.FullPath{"/buckets/b/src/a.jpg", "/buckets/b/src"} {
+		t.Run(string(src), func(t *testing.T) {
+			client := copyingRenameRemoteClient{&renameRemoteClient{}}
+			server, store := newRenameRemoteTestServer(t, client)
+			store.entries["/buckets/b/src/a.jpg"] = remoteOnlyRenameEntry("/buckets/b/src/a.jpg", 101)
+			store.commitErr = errors.New("commit failed")
 
-	err := renameFile(server, "/buckets/b/src/a.jpg", "/buckets/b/dst/a.jpg")
-	require.ErrorContains(t, err, "commit failed")
+			err := renameFile(server, src, "/buckets/b/moved")
+			require.ErrorContains(t, err, "commit failed")
 
-	// remote changes are not rolled back with the store: the old object is
-	// gone, but only after the copy at the new key was written
-	assert.Equal(t, []string{"copy origin/src/a.jpg origin/dst/a.jpg", "delete origin/src/a.jpg"}, client.recorded())
+			// the old object, and the old directory holding it, are deleted
+			// only once the rename has committed
+			dst := "origin/moved"
+			if src == "/buckets/b/src" {
+				dst = "origin/moved/a.jpg"
+			}
+			assert.Equal(t, []string{"copy origin/src/a.jpg " + dst}, client.recorded())
+		})
+	}
 }
 
 func TestRenameEntryWithLocalDataDoesNotCopy(t *testing.T) {
@@ -267,11 +275,13 @@ func TestRenameDirectoryCopiesRemoteOnlyChildren(t *testing.T) {
 
 	require.NoError(t, renameFile(server, "/buckets/b/src", "/buckets/b/moved"))
 
+	// b.jpg's old object goes with its entry; the old directory and a.jpg's
+	// old object wait for the commit
 	assert.Equal(t, []string{
 		"copy origin/src/a.jpg origin/moved/a.jpg",
-		"delete origin/src/a.jpg",
 		"delete origin/src/b.jpg",
 		"rmdir origin/src",
+		"delete origin/src/a.jpg",
 	}, client.recorded())
 	moved, err := store.FindEntry(context.Background(), "/buckets/b/moved/a.jpg")
 	require.NoError(t, err)