seaweedfs/weed/shell/command_remote_uncache.go

package shell

import (
	"context"
	"flag"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"time"

	"github.com/seaweedfs/seaweedfs/weed/filer"
	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
	"github.com/seaweedfs/seaweedfs/weed/util"
	"google.golang.org/protobuf/proto"
)

func init() {
	Commands = append(Commands, &commandRemoteUncache{})
}

type commandRemoteUncache struct {
}

func (c *commandRemoteUncache) Name() string {
	return "remote.uncache"
}

func (c *commandRemoteUncache) Help() string {
	return `keep the metadata but remote cache the file content for mounted directories or files

	This is designed to run regularly. So you can add it to some cronjob.
	If a file is not synchronized with the remote copy, the file will be skipped to avoid loss of data.

	remote.uncache -dir=/xxx
	remote.uncache -dir=/xxx/some/sub/dir
	remote.uncache -dir=/xxx/some/sub/dir -include=*.pdf
	remote.uncache -dir=/xxx/some/sub/dir -exclude=*.txt
	remote.uncache -minSize=1024000    # uncache files larger than 100K
	remote.uncache -minAge=3600        # uncache files older than 1 hour (created time)
	remote.uncache -minCacheAge=3600   # uncache files older than 1 hour (cached time)

`
}

func (c *commandRemoteUncache) HasTag(CommandTag) bool {
	return false
}

func (c *commandRemoteUncache) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	remoteUncacheCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)

	dir := remoteUncacheCommand.String("dir", "", "a directory in filer")
	fileFiler := newFileFilter(remoteUncacheCommand)

	if err = remoteUncacheCommand.Parse(args); err != nil {
		return err
	}

	mappings, listErr := filer.ReadMountMappings(commandEnv.option.GrpcDialOption, commandEnv.FilerAddress())
	if listErr != nil {
		return listErr
	}
	if *dir != "" {
		var localMountedDir string
		for k := range mappings.Mappings {
			if strings.HasPrefix(*dir, k) {
				localMountedDir = k
			}
		}
		if localMountedDir == "" {
			jsonPrintln(writer, mappings)
			fmt.Fprintf(writer, "%s is not mounted\n", *dir)
			return nil
		}

		// pull content from remote
		if err = c.uncacheContentData(commandEnv, writer, util.FullPath(*dir), fileFiler); err != nil {
			return fmt.Errorf("uncache content data: %w", err)
		}
		return nil
	}

	for key, _ := range mappings.Mappings {
		if err := c.uncacheContentData(commandEnv, writer, util.FullPath(key), fileFiler); err != nil {
			return err
		}
	}

	return nil
}

func (c *commandRemoteUncache) uncacheContentData(commandEnv *CommandEnv, writer io.Writer, dirToCache util.FullPath, fileFilter *FileFilter) error {

	return recursivelyTraverseDirectory(commandEnv, dirToCache, func(dir util.FullPath, entry *filer_pb.Entry) bool {

		if !mayHaveCachedToLocal(entry) {
			return true // true means recursive traversal should continue
		}

		if !fileFilter.matches(entry) {
			return true
		}

		if entry.RemoteEntry.LastLocalSyncTsNs < entry.Attributes.Mtime*1e9+int64(entry.Attributes.MtimeNs) {
			return true // should not uncache an entry that is not synchronized with remote
		}

		fmt.Fprintf(writer, "Uncache %+v ... ", dir.Child(entry.Name))

		var stale bool
		err := commandEnv.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
			var updateErr error
			stale, updateErr = uncacheEntry(client, string(dir), entry)
			return updateErr
		})
		if stale {
			fmt.Fprintf(writer, "skipped, changed since it was read\n")
			return true
		}
		if err != nil {
			fmt.Fprintf(writer, "uncache %+v: %v\n", dir.Child(entry.Name), err)
			return false
		}
		fmt.Fprintf(writer, "Done\n")

		return true
	})
}

// uncacheEntry drops the local copy of entry only while the filer still holds
// it as read, so a write that landed after the read keeps its data.
func uncacheEntry(client filer_pb.SeaweedFilerClient, dir string, entry *filer_pb.Entry) (stale bool, err error) {
	readEntry := proto.Clone(entry).(*filer_pb.Entry)
	entry.RemoteEntry.LastLocalSyncTsNs = 0
	entry.Chunks = nil
	return updateEntryIfUnchanged(context.Background(), client, dir, entry, readEntry)
}

type FileFilter struct {
	include     *string
	exclude     *string
	minSize     *int64
	maxSize     *int64
	minAge      *int64
	maxAge      *int64
	minCacheAge *int64
	now         int64
}

func newFileFilter(remoteMountCommand *flag.FlagSet) (ff *FileFilter) {
	ff = &FileFilter{}
	ff.include = remoteMountCommand.String("include", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.exclude = remoteMountCommand.String("exclude", "", "patterns of file names, e.g., *.pdf, *.html, ab?d.txt")
	ff.minSize = remoteMountCommand.Int64("minSize", -1, "minimum file size in bytes")
	ff.maxSize = remoteMountCommand.Int64("maxSize", -1, "maximum file size in bytes")
	ff.minAge = remoteMountCommand.Int64("minAge", -1, "minimum file age in seconds (created time)")
	ff.maxAge = remoteMountCommand.Int64("maxAge", -1, "maximum file age in seconds (created time)")
	ff.minCacheAge = remoteMountCommand.Int64("minCacheAge", -1, "minimum file cache age in seconds (last cached time)")
	ff.now = time.Now().Unix()
	return
}

// matchesName applies only the name-based include/exclude patterns,
// usable for remote entries where local attributes are not available.
func (ff *FileFilter) matchesName(name string) bool {
	if *ff.include != "" {
		if ok, _ := filepath.Match(*ff.include, name); !ok {
			return false
		}
	}
	if *ff.exclude != "" {
		if ok, _ := filepath.Match(*ff.exclude, name); ok {
			return false
		}
	}
	return true
}

func (ff *FileFilter) matches(entry *filer_pb.Entry) bool {
	if entry.Attributes == nil {
		return false
	}
	if !ff.matchesName(entry.Name) {
		return false
	}
	if *ff.minSize != -1 {
		if int64(entry.Attributes.FileSize) < *ff.minSize {
			return false
		}
	}
	if *ff.maxSize != -1 {
		if int64(entry.Attributes.FileSize) > *ff.maxSize {
			return false
		}
	}
	if *ff.minAge != -1 {
		if entry.Attributes.Crtime+*ff.minAge > ff.now {
			return false
		}
	}
	if *ff.maxAge != -1 {
		if entry.Attributes.Crtime+*ff.maxAge < ff.now {
			return false
		}
	}
	if *ff.minCacheAge != -1 {
		lastCachedTime := entry.Attributes.Crtime
		if entry.RemoteEntry != nil && entry.RemoteEntry.LastLocalSyncTsNs > 0 {
			lastCachedTime = entry.RemoteEntry.LastLocalSyncTsNs / 1e9
		}
		if lastCachedTime+*ff.minCacheAge > ff.now {
			return false
		}
	}
	return true
}