Include meta in ReadAllNeedles (#3991)

This is useful for doing backups on the data so we can accurately store the
last modified time, the compression state, and verify the crc.

Previously we were doing VolumeNeedleStatus and then an HTTP request which
needlessly read from the dat file twice.

unmaintained/stream_read_volume/stream_read_volume.go

@@ -5,13 +5,14 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
+
 	"github.com/seaweedfs/seaweedfs/weed/operation"
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"google.golang.org/grpc"
-	"io"
 )
 
 var (
@@ -29,7 +30,7 @@ func main() {
 	vid := uint32(*volumeId)
 
 	eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
-		fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
+		fmt.Printf("%d,%x%08x %d %v %d %x\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob), resp.NeedleBlobCompressed, resp.LastModified, resp.Crc)
 		return nil
 	}
 

weed/pb/volume_server.proto

@@ -317,6 +317,9 @@ message ReadAllNeedlesResponse {
     uint64 needle_id = 2;
     uint32 cookie = 3;
     bytes needle_blob = 5;
+    bool needle_blob_compressed = 6;
+    uint64 last_modified = 7;
+    uint32 crc = 8;
 }
 
 message VolumeTailSenderRequest {

weed/storage/volume_read_all.go

@@ -30,10 +30,13 @@ func (scanner *VolumeFileScanner4ReadAll) VisitNeedle(n *needle.Needle, offset i
 	}
 
 	sendErr := scanner.Stream.Send(&volume_server_pb.ReadAllNeedlesResponse{
-		VolumeId:   uint32(scanner.V.Id),
-		NeedleId:   uint64(n.Id),
-		Cookie:     uint32(n.Cookie),
-		NeedleBlob: n.Data,
+		VolumeId:             uint32(scanner.V.Id),
+		NeedleId:             uint64(n.Id),
+		Cookie:               uint32(n.Cookie),
+		NeedleBlob:           n.Data,
+		NeedleBlobCompressed: n.IsCompressed(),
+		LastModified:         n.LastModified,
+		Crc:                  n.Checksum.Value(),
 	})
 	if sendErr != nil {
 		return sendErr