preserve Content-Encoding for remote-mounted objects (#10340)

* remote storage: carry Content-Encoding into mounted entries

A RemoteEntry now records the remote object's Content-Encoding, and every
path that materializes a local entry from remote metadata (lazy fetch, lazy
listing, remote.mount, remote.meta.sync, remote.cache) stamps it into the
entry extended attributes, so HTTP and S3 HeadObject/GetObject return the
header. GCS and Azure populate it on listing and stat; S3 only exposes it
via HeadObject, so listings leave it empty.

* remote storage: set Content-Encoding when uploading to the remote

An entry carrying Content-Encoding in its extended attributes (a native S3
upload, or a value pulled from the remote) now keeps it when
filer.remote.sync or remote.copy.local writes the object to GCS, S3, or
Azure, instead of silently dropping it.

* gcs: read remote objects without decompressive transcoding

GCS transparently decompresses gzip-encoded objects on download, which
ignores range requests and returns byte counts that disagree with the
tracked RemoteSize. Request the stored bytes instead; chunked reads of
gzip-encoded objects then behave like any other object.

* remote storage: track Content-Encoding presence so removals propagate

A listing that does not report encodings (S3) leaves the field unset and
the local header untouched, while an authoritative report of no encoding
(GCS, Azure, any stat) now clears a previously stamped header instead of
leaving it stale. remote.cache also schedules a metadata update when only
the reported encoding changes.

* remote storage: propagate Content-Encoding on metadata-only updates

filer.remote.sync routes same-content changes through UpdateFileMetadata,
which only touched custom metadata (GCS, Azure) or tags (S3), so a
Content-Encoding change in the extended attributes never reached the
remote object's real header. GCS now patches contentEncoding alongside
the metadata, and Azure reissues the blob's HTTP headers with the new
value, carrying the others over since the call replaces the full set.
S3 stays tags-only: changing the header there means rewriting the
object, which the sync already does whenever content changes.

* remote.meta.sync: optional per-file stat for listing-omitted metadata

S3 listings carry no Content-Encoding, so entries synced from them never
learn it and the lazy-stat path never runs once an entry exists. With
-statFiles, each new or changed file whose listing left the encoding
unreported is stat-ed before reconciling, and the stat-derived value is
persisted so the next run only stats files that changed. Off by default:
it costs one remote request per file, and GCS and Azure listings already
carry the encoding.

* s3: apply metadata-only Content-Encoding changes with an in-place copy

Content-Encoding is S3 system metadata, so the tags-only metadata update
silently left the object's real header untouched. When the encoding
differs, reissue the object as a self-copy with replaced metadata,
carrying the content type and configured storage class like a fresh
write does. CopyObject caps at 5 GiB; beyond that the change is logged
and applies on the next content write.

* azure: skip the metadata call when user metadata is unchanged

An encoding-only change reissues the blob's HTTP headers; sending the
unchanged user metadata alongside it wastes a round trip and bumps the
blob's ETag once more than needed.

* s3: carry existing object metadata through the encoding copy

The replace directive drops everything not resent, and a mounted entry
usually has no local mime or user metadata, so the in-place copy wiped
the object's Content-Type, Cache-Control, user metadata, encryption
settings, and storage class. Read them back with a HeadObject first and
carry them over, overriding only what SeaweedFS manages: the encoding,
a locally set mime, and the configured storage class. S3 reports
Expires as a string while the copy input wants a time, so it is parsed
and skipped when malformed.

weed/shell/command_remote_meta_sync.go

@@ -43,6 +43,9 @@ func (c *commandRemoteMetaSync) Help() string {
 	Local metadata for files and directories removed from the remote is also
 	removed by default; pass -delete=false to keep it.
 
+	S3 listings do not report Content-Encoding; pass -statFiles to stat each
+	new or changed file so that metadata is synchronized too.
+
 	This is designed to run regularly. So you can add it to some cronjob.
 
 	If there are no other operations changing remote files, this operation is not needed.
@@ -60,6 +63,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
 
 	dir := remoteMetaSyncCommand.String("dir", "", "a directory in filer")
 	deleteStale := remoteMetaSyncCommand.Bool("delete", true, "remove local metadata of files and directories deleted from remote")
+	statFiles := remoteMetaSyncCommand.Bool("statFiles", false, "stat each new or changed file when the listing does not report all metadata (S3 listings lack Content-Encoding); costs one extra remote request per file")
 
 	if err = remoteMetaSyncCommand.Parse(args); err != nil {
 		return nil
@@ -72,7 +76,7 @@ func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer
 	}
 
 	// pull metadata from remote
-	if err = pullMetadata(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *deleteStale); err != nil {
+	if err = pullMetadata(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *deleteStale, *statFiles); err != nil {
 		return fmt.Errorf("cache meta data: %w", err)
 	}
 
@@ -148,7 +152,7 @@ type remoteChild struct {
 	remoteEntry *filer_pb.RemoteEntry
 }
 
-func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf, deleteStale bool) error {
+func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf, deleteStale bool, statFiles bool) error {
 
 	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
 	if err != nil {
@@ -158,7 +162,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util
 	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToCache)
 
 	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
-		return pullMetadataDirectory(context.Background(), client, writer, remoteStorage, dirToCache, remote, deleteStale)
+		return pullMetadataDirectory(context.Background(), client, writer, remoteStorage, dirToCache, remote, deleteStale, statFiles)
 	})
 }
 
@@ -167,7 +171,7 @@ func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util
 // delimiter surfaces subdirectories, including empty ones, as their own
 // entries, so directories are materialized locally even when they hold no
 // files.
-func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, remoteStorage remote_storage.RemoteStorageClient, localDir util.FullPath, remoteLoc *remote_pb.RemoteStorageLocation, deleteStale bool) error {
+func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, remoteStorage remote_storage.RemoteStorageClient, localDir util.FullPath, remoteLoc *remote_pb.RemoteStorageLocation, deleteStale bool, statFiles bool) error {
 
 	remoteChildren := make(map[string]*remoteChild)
 	if err := remoteStorage.ListDirectory(ctx, remoteLoc, func(dir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
@@ -218,6 +222,23 @@ func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClie
 			existingEntry = nil
 		}
 
+		// enrich a listing that does not report encodings with a per-file stat,
+		// skipping files whose stored metadata is already present and unchanged
+		if statFiles && !child.isDirectory && child.remoteEntry.RemoteContentEncoding == nil {
+			needsStat := existingEntry == nil
+			if existingEntry != nil && existingEntry.RemoteEntry != nil {
+				needsStat = existingEntry.RemoteEntry.RemoteContentEncoding == nil ||
+					existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
+					existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
+					existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize
+			}
+			if needsStat {
+				if statEntry, statErr := remoteStorage.StatFile(childRemoteLocation(remoteLoc, name)); statErr == nil && statEntry != nil {
+					child.remoteEntry = statEntry
+				}
+			}
+		}
+
 		if existingEntry == nil {
 			if err := createRemoteEntry(ctx, client, writer, localDir, name, child, remoteLoc.Name); err != nil {
 				return err
@@ -228,7 +249,12 @@ func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClie
 				fmt.Fprintf(writer, "%s (skip)\n", localPath)
 			} else if existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
 				existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
-				existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize {
+				existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize ||
+				(child.remoteEntry.RemoteContentEncoding != nil &&
+					existingEntry.RemoteEntry.GetRemoteContentEncoding() != child.remoteEntry.GetRemoteContentEncoding()) ||
+				// persist the stat-derived encoding so the next run skips the stat
+				(statFiles && existingEntry.RemoteEntry.RemoteContentEncoding == nil &&
+					child.remoteEntry.RemoteContentEncoding != nil) {
 				fmt.Fprintf(writer, "%s (update)\n", localPath)
 				if err := doSaveRemoteEntry(client, string(localDir), existingEntry, child.remoteEntry); err != nil {
 					return err
@@ -239,7 +265,7 @@ func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClie
 		}
 
 		if child.isDirectory {
-			if err := pullMetadataDirectory(ctx, client, writer, remoteStorage, localPath, childRemoteLocation(remoteLoc, name), deleteStale); err != nil {
+			if err := pullMetadataDirectory(ctx, client, writer, remoteStorage, localPath, childRemoteLocation(remoteLoc, name), deleteStale, statFiles); err != nil {
 				return err
 			}
 		}
@@ -341,6 +367,7 @@ func createRemoteEntry(ctx context.Context, client filer_pb.SeaweedFilerClient,
 			Name:        name,
 			IsDirectory: child.isDirectory,
 			Attributes:  attributes,
+			Extended:    filer.MergeRemoteContentEncoding(remoteEntry, nil),
 			RemoteEntry: remoteEntry,
 		},
 	})