filer: delete a copied rename's old object only after the commit

Review follow-ups:

- The old key of a copied remote-only entry, and the old directory of a
  rename that copied one, are now deleted once the rename has committed.
  A rename that fails after the copy leaves the source entry and its
  object in place, plus an extra object at the new key.
- Every part of a multipart copy is conditioned on the ETag the copy
  started from, so an object overwritten mid-copy fails the copy
  instead of mixing versions.
- Both copy paths keep the source's server-side encryption settings.
- filer.remote.sync no longer deletes a key that a remote-only entry
  holds by the time the delete event is processed: a file deleted and
  then replaced by a remote-only rename has its content only there.

weed/remote_storage/s3/s3_storage_client.go

@@ -690,11 +690,14 @@ func (s *s3RemoteStorageClient) CopyFile(src *remote_pb.RemoteStorageLocation, d
 	}
 
 	if size := aws.Int64Value(headOut.ContentLength); size <= s3CopyObjectSizeLimit {
+		// a copy takes the bucket's default encryption unless told otherwise
 		if _, err = s.conn.CopyObject(&s3.CopyObjectInput{
-			Bucket:       aws.String(dst.Bucket),
-			Key:          aws.String(dstKey),
-			CopySource:   copySource,
-			StorageClass: storageClass,
+			Bucket:               aws.String(dst.Bucket),
+			Key:                  aws.String(dstKey),
+			CopySource:           copySource,
+			StorageClass:         storageClass,
+			ServerSideEncryption: headOut.ServerSideEncryption,
+			SSEKMSKeyId:          headOut.SSEKMSKeyId,
 		}); err != nil {
 			return nil, fmt.Errorf("copy s3 %s/%s to %s/%s: %w", src.Bucket, srcKey, dst.Bucket, dstKey, err)
 		}
@@ -744,6 +747,8 @@ func (s *s3RemoteStorageClient) multipartCopy(headOut *s3.HeadObjectOutput, src
 		ContentDisposition:      headOut.ContentDisposition,
 		ContentLanguage:         headOut.ContentLanguage,
 		WebsiteRedirectLocation: headOut.WebsiteRedirectLocation,
+		ServerSideEncryption:    headOut.ServerSideEncryption,
+		SSEKMSKeyId:             headOut.SSEKMSKeyId,
 		StorageClass:            storageClass,
 	}
 	if headOut.Expires != nil {
@@ -781,6 +786,9 @@ func (s *s3RemoteStorageClient) multipartCopy(headOut *s3.HeadObjectOutput, src
 			PartNumber:      aws.Int64(partNumber),
 			CopySource:      copySource,
 			CopySourceRange: aws.String(fmt.Sprintf("bytes=%d-%d", offset, min(offset+partSize, size)-1)),
+			// every part from the object the HeadObject saw, not a mix of
+			// versions if it is overwritten mid-copy
+			CopySourceIfMatch: headOut.ETag,
 		})
 		if partErr == nil && partOut.CopyPartResult == nil {
 			partErr = errors.New("no part result")

weed/remote_storage/s3/s3_storage_client_copy_test.go

@@ -38,6 +38,8 @@ func (m *copyObjectMock) HeadObject(input *awss3.HeadObjectInput) (*awss3.HeadOb
 		ContentType:             aws.String("image/jpeg"),
 		Expires:                 aws.String("Wed, 21 Oct 2026 07:28:00 GMT"),
 		WebsiteRedirectLocation: aws.String("/elsewhere"),
+		ServerSideEncryption:    aws.String("aws:kms"),
+		SSEKMSKeyId:             aws.String("key-1"),
 		ETag:                    aws.String(`"etag-` + aws.StringValue(input.Key) + `"`),
 	}, nil
 }
@@ -100,6 +102,8 @@ func TestS3CopyFileUsesCopyObjectUpToTheLimit(t *testing.T) {
 	require.Len(t, mock.copyInputs, 1)
 	require.Equal(t, "bucket/src/a%20b.jpg", aws.StringValue(mock.copyInputs[0].CopySource))
 	require.Equal(t, "dst/a b.jpg", aws.StringValue(mock.copyInputs[0].Key))
+	require.Equal(t, "aws:kms", aws.StringValue(mock.copyInputs[0].ServerSideEncryption))
+	require.Equal(t, "key-1", aws.StringValue(mock.copyInputs[0].SSEKMSKeyId))
 	require.Nil(t, mock.createInput)
 	// the RemoteEntry describes the destination
 	require.Equal(t, `"etag-dst/a b.jpg"`, remoteEntry.RemoteETag)
@@ -117,6 +121,8 @@ func TestS3CopyFileCopiesLargeObjectsInParts(t *testing.T) {
 	require.NotNil(t, mock.createInput)
 	require.Equal(t, "image/jpeg", aws.StringValue(mock.createInput.ContentType))
 	require.Equal(t, "/elsewhere", aws.StringValue(mock.createInput.WebsiteRedirectLocation))
+	require.Equal(t, "aws:kms", aws.StringValue(mock.createInput.ServerSideEncryption))
+	require.Equal(t, "key-1", aws.StringValue(mock.createInput.SSEKMSKeyId))
 	require.Equal(t, int64(1792567680), aws.TimeValue(mock.createInput.Expires).Unix())
 	// tags only where the remote supports tagging
 	require.Empty(t, mock.tagInputs)
@@ -124,6 +130,9 @@ func TestS3CopyFileCopiesLargeObjectsInParts(t *testing.T) {
 	require.Len(t, mock.partInputs, 11)
 	require.Equal(t, "bytes=0-536870911", aws.StringValue(mock.partInputs[0].CopySourceRange))
 	require.Equal(t, "bytes=5368709120-5368709120", aws.StringValue(mock.partInputs[10].CopySourceRange))
+	for _, part := range mock.partInputs {
+		require.Equal(t, `"etag-src/a b.jpg"`, aws.StringValue(part.CopySourceIfMatch))
+	}
 	require.Len(t, mock.completed.MultipartUpload.Parts, 11)
 	require.Equal(t, int64(11), aws.Int64Value(mock.completed.MultipartUpload.Parts[10].PartNumber))
 	require.False(t, mock.aborted)