Have volume scrubs account for zero-sized volumes. (#9609)

Fixes scrubbing for pre-allocated volumes with zero-size indeces by
reworking the validation code to allow zero-size indeces on zero-size
volumes.
This commit is contained in:
Lisandro Pin authored and GitHub committed 2026-05-21 09:42:07 -07:00
1 parent 3825035f07
commit 39e9294907
4 files changed
+68 -11

No files matched your search

Binary file not shown.
Whitespace-only changes.
+19 -8
View File
@@ -27,9 +27,20 @@ func (v *Volume) openIndex() (*os.File, int64, error) {
idxFile.Close()
return nil, 0, fmt.Errorf("failed to stat IDX file %s for volume %v: %v", idxFileName, v.Id, err)
}
if idxStat.Size() == 0 {
idxFile.Close()
return nil, 0, fmt.Errorf("zero-size IDX file for volume %v at %s", v.Id, idxFileName)
volumeFileSize, _, err := v.DataBackend.GetStat()
if err != nil {
idxFile.Close()
return nil, 0, fmt.Errorf("failed to stat storage for zero-size IDX volume %v", v.Id)
}
// account for pre-allocated volumes (f.ex. after running "volume.grow") without data, as these
// are allowed to have zero-size indeces.
if volumeFileSize > int64(super_block.SuperBlockSize) {
idxFile.Close()
return nil, 0, fmt.Errorf("zero-size IDX file for volume %v with store size %d", v.Id, volumeFileSize)
}
}
return idxFile, idxStat.Size(), nil
@@ -50,7 +61,7 @@ func (v *Volume) ScrubIndex() (int64, []error) {
}
// scrubVolumeData checks a volume content + index for issues.
func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *os.File, idxFileSize int64) (int64, []error) {
func (v *Volume) scrubVolumeData(idxFile *os.File, idxFileSize int64) (int64, []error) {
// full scrubbing means also scrubbing the index
var count int64
_, errs := idx.CheckIndexFile(idxFile, idxFileSize, v.Version())
@@ -73,8 +84,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o
}
n := needle.Needle{}
if err := n.ReadData(dataFile, offset.ToActualOffset(), size, version); err != nil {
errs = append(errs, fmt.Errorf("needle %d on volume %d: %v", id, v.Id, err))
if err := n.ReadData(v.DataBackend, offset.ToActualOffset(), size, version); err != nil {
errs = append(errs, fmt.Errorf("failed to read needle %d on volume %d: %v", id, v.Id, err))
}
totalRead += actualSize
@@ -86,7 +97,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o
// check total volume file size
wantSize := totalRead + super_block.SuperBlockSize
dataSize, _, err := dataFile.GetStat()
dataSize, _, err := v.DataBackend.GetStat()
if err != nil {
errs = append(errs, fmt.Errorf("failed to stat data file for volume %d: %v", v.Id, err))
} else {
@@ -111,7 +123,7 @@ func (v *Volume) Scrub() (int64, []error) {
}
defer idxFile.Close()
return v.scrubVolumeData(v.DataBackend, idxFile, idxFileSize)
return v.scrubVolumeData(idxFile, idxFileSize)
}
func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, err error) {
@@ -175,7 +187,6 @@ func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) (
return lastAppendAtNs, nil
}
func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) {
if indexSize, err = util.GetFileSize(indexFile); err == nil {
if indexSize%types.NeedleMapEntrySize != 0 {
+49 -3
View File
@@ -30,6 +30,51 @@ func TestScrubVolumeData(t *testing.T) {
want: 27,
wantErrs: []error{},
},
{
name: "zero-size volume without index",
dataPath: "./test_files/empty_volume.dat",
indexPath: "./test_files/empty_volume.idx",
version: needle.Version3,
want: 0,
wantErrs: []error{},
},
{
name: "zero-size volume with index",
dataPath: "./test_files/empty_volume.dat",
indexPath: "./test_files/healthy_volume.idx",
version: needle.Version3,
want: 27,
wantErrs: []error{
fmt.Errorf("failed to read needle 3 on volume 0: EOF"),
fmt.Errorf("failed to read needle 4 on volume 0: EOF"),
fmt.Errorf("failed to read needle 8 on volume 0: EOF"),
fmt.Errorf("failed to read needle 9 on volume 0: EOF"),
fmt.Errorf("failed to read needle 11 on volume 0: EOF"),
fmt.Errorf("failed to read needle 18 on volume 0: EOF"),
fmt.Errorf("failed to read needle 20 on volume 0: EOF"),
fmt.Errorf("failed to read needle 21 on volume 0: EOF"),
fmt.Errorf("failed to read needle 25 on volume 0: EOF"),
fmt.Errorf("failed to read needle 29 on volume 0: EOF"),
fmt.Errorf("failed to read needle 31 on volume 0: EOF"),
fmt.Errorf("failed to read needle 33 on volume 0: EOF"),
fmt.Errorf("failed to read needle 35 on volume 0: EOF"),
fmt.Errorf("failed to read needle 39 on volume 0: EOF"),
fmt.Errorf("failed to read needle 45 on volume 0: EOF"),
fmt.Errorf("failed to read needle 46 on volume 0: EOF"),
fmt.Errorf("failed to read needle 48 on volume 0: EOF"),
fmt.Errorf("data file for volume 0 is smaller (8) than the 27 needles it contains (942856)"),
},
},
{
name: "volume without index",
dataPath: "./test_files/healthy_volume.dat",
indexPath: "./test_files/empty_volume.idx",
version: needle.Version3,
want: 0,
wantErrs: []error{
fmt.Errorf("data file size for volume 0 (942856) doesn't match the size for 0 needles read (8)"),
},
},
{
name: "bitrot volume",
dataPath: "./test_files/bitrot_volume.dat",
@@ -37,8 +82,8 @@ func TestScrubVolumeData(t *testing.T) {
version: needle.Version3,
want: 27,
wantErrs: []error{
fmt.Errorf("needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"),
fmt.Errorf("needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"),
fmt.Errorf("failed to read needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"),
fmt.Errorf("failed to read needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"),
fmt.Errorf("data file size for volume 0 (942864) doesn't match the size for 27 needles read (942856)"),
},
},
@@ -64,12 +109,13 @@ func TestScrubVolumeData(t *testing.T) {
}
v := Volume{
DataBackend: backend.NewDiskFile(datFile),
volumeInfo: &volume_server_pb.VolumeInfo{
Version: uint32(tc.version),
},
}
got, gotErrs := v.scrubVolumeData(backend.NewDiskFile(datFile), idxFile, idxStat.Size())
got, gotErrs := v.scrubVolumeData(idxFile, idxStat.Size())
if got != tc.want {
t.Errorf("expected %d files processed, got %d", tc.want, got)