mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-18 01:37:10 +01:00
Have volume scrubs account for zero-sized volumes. (#9609)
Fixes scrubbing for pre-allocated volumes with zero-size indeces by reworking the validation code to allow zero-size indeces on zero-size volumes.
This commit is contained in:
1 parent
3825035f07
commit
39e9294907
4 files changed
+68
-11
No files matched your search
Binary file not shown.
Whitespace-only changes.
@@ -27,9 +27,20 @@ func (v *Volume) openIndex() (*os.File, int64, error) {
|
||||
idxFile.Close()
|
||||
return nil, 0, fmt.Errorf("failed to stat IDX file %s for volume %v: %v", idxFileName, v.Id, err)
|
||||
}
|
||||
|
||||
if idxStat.Size() == 0 {
|
||||
idxFile.Close()
|
||||
return nil, 0, fmt.Errorf("zero-size IDX file for volume %v at %s", v.Id, idxFileName)
|
||||
volumeFileSize, _, err := v.DataBackend.GetStat()
|
||||
if err != nil {
|
||||
idxFile.Close()
|
||||
return nil, 0, fmt.Errorf("failed to stat storage for zero-size IDX volume %v", v.Id)
|
||||
}
|
||||
|
||||
// account for pre-allocated volumes (f.ex. after running "volume.grow") without data, as these
|
||||
// are allowed to have zero-size indeces.
|
||||
if volumeFileSize > int64(super_block.SuperBlockSize) {
|
||||
idxFile.Close()
|
||||
return nil, 0, fmt.Errorf("zero-size IDX file for volume %v with store size %d", v.Id, volumeFileSize)
|
||||
}
|
||||
}
|
||||
|
||||
return idxFile, idxStat.Size(), nil
|
||||
@@ -50,7 +61,7 @@ func (v *Volume) ScrubIndex() (int64, []error) {
|
||||
}
|
||||
|
||||
// scrubVolumeData checks a volume content + index for issues.
|
||||
func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *os.File, idxFileSize int64) (int64, []error) {
|
||||
func (v *Volume) scrubVolumeData(idxFile *os.File, idxFileSize int64) (int64, []error) {
|
||||
// full scrubbing means also scrubbing the index
|
||||
var count int64
|
||||
_, errs := idx.CheckIndexFile(idxFile, idxFileSize, v.Version())
|
||||
@@ -73,8 +84,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o
|
||||
}
|
||||
|
||||
n := needle.Needle{}
|
||||
if err := n.ReadData(dataFile, offset.ToActualOffset(), size, version); err != nil {
|
||||
errs = append(errs, fmt.Errorf("needle %d on volume %d: %v", id, v.Id, err))
|
||||
if err := n.ReadData(v.DataBackend, offset.ToActualOffset(), size, version); err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to read needle %d on volume %d: %v", id, v.Id, err))
|
||||
}
|
||||
|
||||
totalRead += actualSize
|
||||
@@ -86,7 +97,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o
|
||||
|
||||
// check total volume file size
|
||||
wantSize := totalRead + super_block.SuperBlockSize
|
||||
dataSize, _, err := dataFile.GetStat()
|
||||
dataSize, _, err := v.DataBackend.GetStat()
|
||||
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to stat data file for volume %d: %v", v.Id, err))
|
||||
} else {
|
||||
@@ -111,7 +123,7 @@ func (v *Volume) Scrub() (int64, []error) {
|
||||
}
|
||||
defer idxFile.Close()
|
||||
|
||||
return v.scrubVolumeData(v.DataBackend, idxFile, idxFileSize)
|
||||
return v.scrubVolumeData(idxFile, idxFileSize)
|
||||
}
|
||||
|
||||
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, err error) {
|
||||
@@ -175,7 +187,6 @@ func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) (
|
||||
return lastAppendAtNs, nil
|
||||
}
|
||||
|
||||
|
||||
func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) {
|
||||
if indexSize, err = util.GetFileSize(indexFile); err == nil {
|
||||
if indexSize%types.NeedleMapEntrySize != 0 {
|
||||
|
||||
@@ -30,6 +30,51 @@ func TestScrubVolumeData(t *testing.T) {
|
||||
want: 27,
|
||||
wantErrs: []error{},
|
||||
},
|
||||
{
|
||||
name: "zero-size volume without index",
|
||||
dataPath: "./test_files/empty_volume.dat",
|
||||
indexPath: "./test_files/empty_volume.idx",
|
||||
version: needle.Version3,
|
||||
want: 0,
|
||||
wantErrs: []error{},
|
||||
},
|
||||
{
|
||||
name: "zero-size volume with index",
|
||||
dataPath: "./test_files/empty_volume.dat",
|
||||
indexPath: "./test_files/healthy_volume.idx",
|
||||
version: needle.Version3,
|
||||
want: 27,
|
||||
wantErrs: []error{
|
||||
fmt.Errorf("failed to read needle 3 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 4 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 8 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 9 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 11 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 18 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 20 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 21 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 25 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 29 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 31 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 33 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 35 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 39 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 45 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 46 on volume 0: EOF"),
|
||||
fmt.Errorf("failed to read needle 48 on volume 0: EOF"),
|
||||
fmt.Errorf("data file for volume 0 is smaller (8) than the 27 needles it contains (942856)"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "volume without index",
|
||||
dataPath: "./test_files/healthy_volume.dat",
|
||||
indexPath: "./test_files/empty_volume.idx",
|
||||
version: needle.Version3,
|
||||
want: 0,
|
||||
wantErrs: []error{
|
||||
fmt.Errorf("data file size for volume 0 (942856) doesn't match the size for 0 needles read (8)"),
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bitrot volume",
|
||||
dataPath: "./test_files/bitrot_volume.dat",
|
||||
@@ -37,8 +82,8 @@ func TestScrubVolumeData(t *testing.T) {
|
||||
version: needle.Version3,
|
||||
want: 27,
|
||||
wantErrs: []error{
|
||||
fmt.Errorf("needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"),
|
||||
fmt.Errorf("needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"),
|
||||
fmt.Errorf("failed to read needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"),
|
||||
fmt.Errorf("failed to read needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"),
|
||||
fmt.Errorf("data file size for volume 0 (942864) doesn't match the size for 27 needles read (942856)"),
|
||||
},
|
||||
},
|
||||
@@ -64,12 +109,13 @@ func TestScrubVolumeData(t *testing.T) {
|
||||
}
|
||||
|
||||
v := Volume{
|
||||
DataBackend: backend.NewDiskFile(datFile),
|
||||
volumeInfo: &volume_server_pb.VolumeInfo{
|
||||
Version: uint32(tc.version),
|
||||
},
|
||||
}
|
||||
|
||||
got, gotErrs := v.scrubVolumeData(backend.NewDiskFile(datFile), idxFile, idxStat.Size())
|
||||
got, gotErrs := v.scrubVolumeData(idxFile, idxStat.Size())
|
||||
|
||||
if got != tc.want {
|
||||
t.Errorf("expected %d files processed, got %d", tc.want, got)
|
||||
|
||||
Reference in new issue
Block a user