Merge master and resolve conflicts

weed/shell/command_fs_meta_save.go

@@ -2,6 +2,8 @@ package shell
 
 import (
 	"compress/gzip"
+	"context"
+	"errors"
 	"flag"
 	"fmt"
 	"io"
@@ -146,6 +148,29 @@ func doTraverseBfsAndSaving(filerClient filer_pb.FilerClient, writer io.Writer,
 
 	var dirCount, fileCount uint64
 
+	// also save the directory itself (path) if it exists in the filer
+	if e, getErr := filer_pb.GetEntry(context.Background(), filerClient, util.FullPath(path)); getErr != nil {
+		// Entry not found is expected and can be ignored; log other errors.
+		if !errors.Is(getErr, filer_pb.ErrNotFound) {
+			fmt.Fprintf(writer, "failed to get entry %s: %v\n", path, getErr)
+		}
+	} else if e != nil {
+		parentDir, _ := util.FullPath(path).DirAndName()
+		protoMessage := &filer_pb.FullEntry{
+			Dir:   parentDir,
+			Entry: e,
+		}
+		if genErr := genFn(protoMessage, outputChan); genErr != nil {
+			fmt.Fprintf(writer, "marshall error: %v\n", genErr)
+		} else {
+			if e.IsDirectory {
+				atomic.AddUint64(&dirCount, 1)
+			} else {
+				atomic.AddUint64(&fileCount, 1)
+			}
+		}
+	}
+
 	err := filer_pb.TraverseBfs(filerClient, util.FullPath(path), func(parentPath util.FullPath, entry *filer_pb.Entry) {
 
 		if strings.HasPrefix(string(parentPath), filer.SystemLogDir) {

weed/shell/command_remote_configure.go

@@ -4,15 +4,16 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
 	"github.com/seaweedfs/seaweedfs/weed/remote_storage"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/protobuf/proto"
-	"io"
-	"regexp"
-	"strings"
 )
 
 func init() {
@@ -34,6 +35,7 @@ func (c *commandRemoteConfigure) Help() string {
 
 	# set or update a configuration
 	remote.configure -name=cloud1 -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2
+	remote.configure -name=cloud1_deep -type=s3 -s3.access_key=xxx -s3.secret_key=yyy -s3.region=us-east-2 -s3.storage_class=DEEP_ARCHIVE
 	remote.configure -name=cloud2 -type=gcs -gcs.appCredentialsFile=~/service-account-file.json -gcs.projectId=yyy
 	remote.configure -name=cloud3 -type=azure -azure.account_name=xxx -azure.account_key=yyy
 	remote.configure -name=cloud4 -type=aliyun -aliyun.access_key=xxx -aliyun.secret_key=yyy -aliyun.endpoint=oss-cn-shenzhen.aliyuncs.com -aliyun.region=cn-sehnzhen

weed/shell/command_volume_fix_replication.go

@@ -69,6 +69,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
 	applyChanges := volFixReplicationCommand.Bool("apply", false, "apply the fix")
 	// TODO: remove this alias
 	applyChangesAlias := volFixReplicationCommand.Bool("force", false, "apply the fix (alias for -apply)")
+	verbose := volFixReplicationCommand.Bool("verbose", false, "show volumes being checked and their statuses")
 	doDelete := volFixReplicationCommand.Bool("doDelete", true, "Also delete over-replicated volumes besides fixing under-replication")
 	doCheck := volFixReplicationCommand.Bool("doCheck", true, "Also check synchronization before deleting")
 	maxParallelization := volFixReplicationCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
@@ -93,6 +94,9 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
 		fixedVolumeReplicas := map[string]int{}
 
 		// collect topology information
+		if *verbose {
+			fmt.Fprintf(writer, "wait 15 seconds and then collect topology information...\n")
+		}
 		topologyInfo, _, err := collectTopologyInfo(commandEnv, 15*time.Second)
 		if err != nil {
 			return err
@@ -102,6 +106,10 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
 		// collect all data nodes
 		volumeReplicas, allLocations := collectVolumeReplicaLocations(topologyInfo)
 
+		if *verbose {
+			fmt.Fprintf(writer, "collected topology: %d locations, %d volumes to check\n", len(allLocations), len(volumeReplicas))
+		}
+
 		if len(allLocations) == 0 {
 			return fmt.Errorf("no data nodes at all")
 		}
@@ -111,16 +119,23 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
 		for vid, replicas := range volumeReplicas {
 			replica := replicas[0]
 			replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(replica.info.ReplicaPlacement))
+
+			// build locations list for optional verbose output
+			locations := make([]string, 0, len(replicas))
+			for _, r := range replicas {
+				locations = append(locations, r.location.String())
+			}
+
+			if *verbose {
+				fmt.Fprintf(writer, "checking volume %d replication %s has %d replicas [%s]\n", replica.info.Id, replicaPlacement, len(replicas), strings.Join(locations, ", "))
+			}
+
 			switch {
 			case replicaPlacement.GetCopyCount() > len(replicas) || !satisfyReplicaCurrentLocation(replicaPlacement, replicas):
 				underReplicatedVolumeIds = append(underReplicatedVolumeIds, vid)
 				fmt.Fprintf(writer, "volume %d replication %s, but under replicated %+d\n", replica.info.Id, replicaPlacement, len(replicas))
 			case isMisplaced(replicas, replicaPlacement):
 				misplacedVolumeIds = append(misplacedVolumeIds, vid)
-				locations := make([]string, 0, len(replicas))
-				for _, r := range replicas {
-					locations = append(locations, r.location.String())
-				}
 				fmt.Fprintf(writer, "volume %d replication %s is not well placed [%s]\n", replica.info.Id, replicaPlacement, strings.Join(locations, ", "))
 			case replicaPlacement.GetCopyCount() < len(replicas):
 				overReplicatedVolumeIds = append(overReplicatedVolumeIds, vid)