seaweedfs/weed/shell/command_remote_meta_sync.go

package shell

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
	"github.com/seaweedfs/seaweedfs/weed/util"
)

func init() {
	Commands = append(Commands, &commandRemoteMetaSync{})
}

type commandRemoteMetaSync struct {
}

func (c *commandRemoteMetaSync) Name() string {
	return "remote.meta.sync"
}

func (c *commandRemoteMetaSync) Help() string {
	return `synchronize the local file meta data with the remote file metadata

	# assume a remote storage is configured to name "cloud1"
	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy
	# mount and pull one bucket
	remote.mount -dir=/xxx -remote=cloud1/bucket

	After mount, if the remote file can be changed, 
	run this command to synchronize the metadata of the mounted folder or any sub folder

		remote.meta.sync -dir=/xxx
		remote.meta.sync -dir=/xxx/some/subdir

	Local metadata for files and directories removed from the remote is also
	removed by default; pass -delete=false to keep it.

	S3 listings do not report Content-Encoding; pass -statFiles to stat each
	new or changed file so that metadata is synchronized too.

	This is designed to run regularly. So you can add it to some cronjob.

	If there are no other operations changing remote files, this operation is not needed.

`
}

func (c *commandRemoteMetaSync) HasTag(CommandTag) bool {
	return false
}

func (c *commandRemoteMetaSync) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteMetaSyncCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteMetaSyncCommand.String("dir", "", "a directory in filer")
	deleteStale := remoteMetaSyncCommand.Bool("delete", true, "remove local metadata of files and directories deleted from remote")
	statFiles := remoteMetaSyncCommand.Bool("statFiles", false, "stat each new or changed file when the listing does not report all metadata (S3 listings lack Content-Encoding); costs one extra remote request per file")

	if err = remoteMetaSyncCommand.Parse(args); err != nil {
		return err
	}

	mappings, localMountedDir, remoteStorageMountedLocation, remoteStorageConf, detectErr := detectMountInfo(commandEnv, writer, *dir)
	if detectErr != nil {
		jsonPrintln(writer, mappings)
		return detectErr
	}

	// pull metadata from remote
	if err = pullMetadata(commandEnv, writer, util.FullPath(localMountedDir), remoteStorageMountedLocation, util.FullPath(*dir), remoteStorageConf, *deleteStale, *statFiles); err != nil {
		return fmt.Errorf("cache meta data: %w", err)
	}

	return nil
}

func detectMountInfo(commandEnv *CommandEnv, writer io.Writer, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) {
	return filer.DetectMountInfo(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress(), dir)
}

/*
This function update entry.RemoteEntry if the remote has any changes.

To pull remote updates, or created for the first time, the criteria is:

	entry == nil or (entry.RemoteEntry != nil and (entry.RemoteEntry.RemoteTag != remote.RemoteTag or entry.RemoteEntry.RemoteMTime < remote.RemoteMTime ))

After the meta pull, the entry.RemoteEntry will have:

	remoteEntry.LastLocalSyncTsNs == 0
	Attributes.FileSize = uint64(remoteEntry.RemoteSize)
	Attributes.Mtime = remoteEntry.RemoteMtime
	remoteEntry.RemoteTag   = actual remote tag
	chunks = nil

When reading the file content or pulling the file content in "remote.cache", the criteria is:

	Attributes.FileSize > 0 and len(chunks) == 0

After caching the file content, the entry.RemoteEntry will be

	remoteEntry.LastLocalSyncTsNs == time.Now.UnixNano()
	Attributes.FileSize = uint64(remoteEntry.RemoteSize)
	Attributes.Mtime = remoteEntry.RemoteMtime
	chunks = non-empty

When "weed filer.remote.sync" to upload local changes to remote, the criteria is:

	Attributes.Mtime > remoteEntry.RemoteMtime

Right after "weed filer.remote.sync", the entry.RemoteEntry will be

	remoteEntry.LastLocalSyncTsNs = time.Now.UnixNano()
	remoteEntry.RemoteSize  = actual remote size, which should equal to entry.Attributes.FileSize
	remoteEntry.RemoteMtime = actual remote mtime, which should be a little greater than entry.Attributes.Mtime
	remoteEntry.RemoteTag   = actual remote tag

If entry does not exist, need to pull meta
If entry.RemoteEntry == nil, this is a new local change and should not be overwritten

	If entry.RemoteEntry.RemoteTag != remoteEntry.RemoteTag {
	  the remote version is updated, need to pull meta
	}
*/

// remoteEntryFileMode returns the POSIX mode for an entry synced from a remote
// whose listing carries no mode. It matches what SeaweedFS S3 assigns native
// objects (files 0660) so a mounted bucket matches the source, deriving the
// directory mode with the same 0111 traversal mask the filer uses for
// auto-created parents (0660 -> 0771).
func remoteEntryFileMode(isDirectory bool) uint32 {
	mode := uint32(0660)
	if isDirectory {
		mode = uint32(os.ModeDir) | mode | 0111
	}
	return mode
}

// remoteChild is one entry of a single remote directory level: a file carries
// its RemoteEntry, a directory carries none.
type remoteChild struct {
	isDirectory bool
	remoteEntry *filer_pb.RemoteEntry
}

func pullMetadata(commandEnv *CommandEnv, writer io.Writer, localMountedDir util.FullPath, remoteMountedLocation *remote_pb.RemoteStorageLocation, dirToCache util.FullPath, remoteConf *remote_pb.RemoteConf, deleteStale bool, statFiles bool) error {

	remoteStorage, err := remote_storage.GetRemoteStorage(remoteConf)
	if err != nil {
		return err
	}

	remote := filer.MapFullPathToRemoteStorageLocation(localMountedDir, remoteMountedLocation, dirToCache)

	return commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
		return pullMetadataDirectory(context.Background(), client, writer, remoteStorage, dirToCache, remote, deleteStale, statFiles)
	})
}

// pullMetadataDirectory reconciles one local directory with its remote
// counterpart and recurses into every remote subdirectory. Listing with a
// delimiter surfaces subdirectories, including empty ones, as their own
// entries, so directories are materialized locally even when they hold no
// files.
func pullMetadataDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, remoteStorage remote_storage.RemoteStorageClient, localDir util.FullPath, remoteLoc *remote_pb.RemoteStorageLocation, deleteStale bool, statFiles bool) error {

	remoteChildren := make(map[string]*remoteChild)
	if err := remoteStorage.ListDirectory(ctx, remoteLoc, func(dir, name string, isDirectory bool, remoteEntry *filer_pb.RemoteEntry) error {
		remoteChildren[name] = &remoteChild{isDirectory: isDirectory, remoteEntry: remoteEntry}
		return nil
	}); err != nil {
		return err
	}

	localChildren, err := listLocalDirectory(ctx, client, localDir)
	if err != nil {
		return err
	}

	for name, child := range remoteChildren {
		localPath := localDir.Child(name)
		existingEntry := localChildren[name]

		if existingEntry != nil && existingEntry.IsDirectory != child.isDirectory {
			if existingEntry.RemoteEntry == nil {
				// a local change conflicts with a changed remote type; keep local
				fmt.Fprintf(writer, "%s (skip)\n", localPath)
				continue
			}
			// the remote swapped file for directory (or vice versa); drop the
			// stale entry so it is recreated with the right type, but keep any
			// local-only descendants of a directory
			if existingEntry.IsDirectory {
				if err := deleteRemoteBackedEntry(ctx, client, writer, localDir, name, existingEntry); err != nil {
					return err
				}
				refreshed, err := listLocalDirectory(ctx, client, localDir)
				if err != nil {
					return err
				}
				if refreshed[name] != nil {
					// local-only entries remain, so the directory stays and the
					// remote file cannot take its place
					fmt.Fprintf(writer, "%s (skip)\n", localPath)
					continue
				}
			} else {
				kept, err := deleteLocalEntry(ctx, client, writer, localDir, existingEntry)
				if err != nil {
					return err
				}
				if kept {
					continue
				}
			}
			existingEntry = nil
		}

		// enrich a listing that does not report encodings with a per-file stat,
		// skipping files whose stored metadata is already present and unchanged
		if statFiles && !child.isDirectory && child.remoteEntry.RemoteContentEncoding == nil {
			needsStat := existingEntry == nil
			if existingEntry != nil && existingEntry.RemoteEntry != nil {
				needsStat = existingEntry.RemoteEntry.RemoteContentEncoding == nil ||
					existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
					existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
					existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize
			}
			if needsStat {
				if statEntry, statErr := remoteStorage.StatFile(childRemoteLocation(remoteLoc, name)); statErr == nil && statEntry != nil {
					child.remoteEntry = statEntry
				}
			}
		}

		if existingEntry == nil {
			taken, err := createRemoteEntry(ctx, client, writer, localDir, name, child, remoteLoc.Name)
			if err != nil {
				return err
			}
			if taken && child.isDirectory {
				// descending into a name a local file took would turn it into a directory
				resp, err := filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{Directory: string(localDir), Name: name})
				if err != nil && !errors.Is(err, filer_pb.ErrNotFound) {
					return err
				}
				if resp == nil || !resp.Entry.IsDirectory {
					continue
				}
			}
		} else if !child.isDirectory {
			if existingEntry.RemoteEntry == nil {
				// a local change that should not be overwritten
				fmt.Fprintf(writer, "%s (skip)\n", localPath)
			} else if existingEntry.RemoteEntry.RemoteETag != child.remoteEntry.RemoteETag ||
				existingEntry.RemoteEntry.RemoteMtime != child.remoteEntry.RemoteMtime ||
				existingEntry.RemoteEntry.RemoteSize != child.remoteEntry.RemoteSize ||
				(child.remoteEntry.RemoteContentEncoding != nil &&
					existingEntry.RemoteEntry.GetRemoteContentEncoding() != child.remoteEntry.GetRemoteContentEncoding()) ||
				// persist the stat-derived encoding so the next run skips the stat
				(statFiles && existingEntry.RemoteEntry.RemoteContentEncoding == nil &&
					child.remoteEntry.RemoteContentEncoding != nil) {
				stale, err := doSaveRemoteEntry(client, string(localDir), existingEntry, child.remoteEntry)
				if stale {
					fmt.Fprintf(writer, "%s (skip, changed since listed)\n", localPath)
				} else if err != nil {
					return err
				} else {
					fmt.Fprintf(writer, "%s (update)\n", localPath)
				}
			} else {
				fmt.Fprintf(writer, "%s (skip)\n", localPath)
			}
		}

		if child.isDirectory {
			if err := pullMetadataDirectory(ctx, client, writer, remoteStorage, localPath, childRemoteLocation(remoteLoc, name), deleteStale, statFiles); err != nil {
				return err
			}
		}
	}

	if deleteStale {
		for name, existingEntry := range localChildren {
			if _, ok := remoteChildren[name]; ok {
				continue
			}
			if err := deleteRemoteBackedEntry(ctx, client, writer, localDir, name, existingEntry); err != nil {
				return err
			}
		}
	}

	return nil
}

// deleteRemoteBackedEntry removes a local entry whose remote source is gone. A
// file is deleted; a directory is descended into locally so its remote-backed
// children are cleaned, and the directory itself is removed only when it was
// remote-backed and holds no remaining local-only entries. Entries without a
// RemoteEntry are local changes and are never touched.
func deleteRemoteBackedEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, localDir util.FullPath, name string, existingEntry *filer_pb.Entry) error {
	localPath := localDir.Child(name)

	if !existingEntry.IsDirectory {
		if existingEntry.RemoteEntry == nil {
			return nil
		}
		_, err := deleteLocalEntry(ctx, client, writer, localDir, existingEntry)
		return err
	}

	children, err := listLocalDirectory(ctx, client, localPath)
	if err != nil {
		return err
	}
	for childName, childEntry := range children {
		if err := deleteRemoteBackedEntry(ctx, client, writer, localPath, childName, childEntry); err != nil {
			return err
		}
	}

	if existingEntry.RemoteEntry == nil {
		return nil
	}
	empty, err := isLocalDirectoryEmpty(ctx, client, localPath)
	if err != nil {
		return err
	}
	if empty {
		_, err = deleteLocalEntry(ctx, client, writer, localDir, existingEntry)
	}
	return err
}

// deleteLocalEntry removes the entry as it was listed; the filer keeps an entry
// whose mtime a local write has moved past the listed one, reported as kept.
// An entry listed without an mtime is kept once a write sets one.
func deleteLocalEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, localDir util.FullPath, entry *filer_pb.Entry) (kept bool, err error) {
	req := &filer_pb.DeleteEntryRequest{Directory: string(localDir), Name: entry.Name, IsDeleteData: true}
	var ok bool
	req.IfNotModifiedAfter, req.IfNotModifiedAfterNs, ok = deleteEntryCutoff(entry.Attributes.GetMtime(), entry.Attributes.GetMtimeNs())
	if !ok {
		req.IfNotModifiedAfterNs = max(1, int64(entry.Attributes.GetMtimeNs()))
	}
	if _, err = filer_pb.DoRemoveRequest(ctx, client, req); err != nil {
		return false, err
	}
	localPath := localDir.Child(entry.Name)
	_, err = filer_pb.LookupEntry(ctx, client, &filer_pb.LookupDirectoryEntryRequest{Directory: string(localDir), Name: entry.Name})
	if err == nil {
		fmt.Fprintf(writer, "%s (skip, changed since listed)\n", localPath)
		return true, nil
	}
	if !errors.Is(err, filer_pb.ErrNotFound) {
		return false, err
	}
	fmt.Fprintf(writer, "%s (delete)\n", localPath)
	return false, nil
}

func isLocalDirectoryEmpty(ctx context.Context, client filer_pb.SeaweedFilerClient, dir util.FullPath) (bool, error) {
	empty := true
	err := filer_pb.SeaweedList(ctx, client, string(dir), "", func(entry *filer_pb.Entry, isLast bool) error {
		empty = false
		return nil
	}, "", false, 1)
	return empty, err
}

// createRemoteEntry creates the entry only if the name is still free; taken
// reports that something was created there after the directory was listed.
func createRemoteEntry(ctx context.Context, client filer_pb.SeaweedFilerClient, writer io.Writer, localDir util.FullPath, name string, child *remoteChild, storageName string) (taken bool, err error) {
	attributes := &filer_pb.FuseAttributes{
		FileMode: remoteEntryFileMode(child.isDirectory),
		TtlSec:   0, // Remote entries should not have TTL
	}
	remoteEntry := child.remoteEntry
	if child.isDirectory {
		// remote listings carry no directory timestamp; stamp with sync time
		now := time.Now().Unix()
		attributes.Crtime = now
		attributes.Mtime = now
		// mark the directory remote-backed so it can be reconciled and removed
		// once it disappears from the remote
		remoteEntry = &filer_pb.RemoteEntry{StorageName: storageName}
	} else {
		attributes.FileSize = uint64(child.remoteEntry.RemoteSize)
		attributes.Mtime = child.remoteEntry.RemoteMtime
	}
	err = filer_pb.CreateEntry(ctx, client, &filer_pb.CreateEntryRequest{
		Directory: string(localDir),
		Entry: &filer_pb.Entry{
			Name:        name,
			IsDirectory: child.isDirectory,
			Attributes:  attributes,
			Extended:    filer.MergeRemoteContentEncoding(remoteEntry, nil),
			RemoteEntry: remoteEntry,
		},
		OExcl: true,
	})
	if errors.Is(err, filer_pb.ErrEntryAlreadyExists) {
		fmt.Fprintf(writer, "%s (skip, created since listed)\n", localDir.Child(name))
		return true, nil
	}
	if err != nil {
		return false, err
	}
	fmt.Fprintf(writer, "%s (create)\n", localDir.Child(name))
	return false, nil
}

func childRemoteLocation(remoteLoc *remote_pb.RemoteStorageLocation, name string) *remote_pb.RemoteStorageLocation {
	return &remote_pb.RemoteStorageLocation{
		Name:   remoteLoc.Name,
		Bucket: remoteLoc.Bucket,
		Path:   string(util.FullPath(remoteLoc.Path).Child(name)),
	}
}

func listLocalDirectory(ctx context.Context, client filer_pb.SeaweedFilerClient, dir util.FullPath) (map[string]*filer_pb.Entry, error) {
	const paginationLimit = 10000
	entries := make(map[string]*filer_pb.Entry)
	lastFileName := ""
	for {
		count := 0
		if err := filer_pb.SeaweedList(ctx, client, string(dir), "", func(entry *filer_pb.Entry, isLast bool) error {
			entries[entry.Name] = entry
			lastFileName = entry.Name
			count++
			return nil
		}, lastFileName, false, paginationLimit); err != nil {
			return nil, err
		}
		if count < paginationLimit {
			return entries, nil
		}
	}
}