From 39e9294907122983354d901b1895f6b93729f208 Mon Sep 17 00:00:00 2001 From: Lisandro Pin Date: Thu, 21 May 2026 18:42:07 +0200 Subject: [PATCH] Have volume scrubs account for zero-sized volumes. (#9609) Fixes scrubbing for pre-allocated volumes with zero-size indeces by reworking the validation code to allow zero-size indeces on zero-size volumes. --- weed/storage/test_files/empty_volume.dat | Bin 0 -> 8 bytes weed/storage/test_files/empty_volume.idx | 0 weed/storage/volume_checking.go | 27 ++++++++---- weed/storage/volume_checking_test.go | 52 +++++++++++++++++++++-- 4 files changed, 68 insertions(+), 11 deletions(-) create mode 100644 weed/storage/test_files/empty_volume.dat create mode 100644 weed/storage/test_files/empty_volume.idx diff --git a/weed/storage/test_files/empty_volume.dat b/weed/storage/test_files/empty_volume.dat new file mode 100644 index 0000000000000000000000000000000000000000..1b0707a54befb13cc5be1694e364095d8aa601b3 GIT binary patch literal 8 LcmZQ(VgLgG04@Lq literal 0 HcmV?d00001 diff --git a/weed/storage/test_files/empty_volume.idx b/weed/storage/test_files/empty_volume.idx new file mode 100644 index 0000000000..e69de29bb2 diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index f398b82e2c..8913f683b5 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -27,9 +27,20 @@ func (v *Volume) openIndex() (*os.File, int64, error) { idxFile.Close() return nil, 0, fmt.Errorf("failed to stat IDX file %s for volume %v: %v", idxFileName, v.Id, err) } + if idxStat.Size() == 0 { - idxFile.Close() - return nil, 0, fmt.Errorf("zero-size IDX file for volume %v at %s", v.Id, idxFileName) + volumeFileSize, _, err := v.DataBackend.GetStat() + if err != nil { + idxFile.Close() + return nil, 0, fmt.Errorf("failed to stat storage for zero-size IDX volume %v", v.Id) + } + + // account for pre-allocated volumes (f.ex. after running "volume.grow") without data, as these + // are allowed to have zero-size indeces. + if volumeFileSize > int64(super_block.SuperBlockSize) { + idxFile.Close() + return nil, 0, fmt.Errorf("zero-size IDX file for volume %v with store size %d", v.Id, volumeFileSize) + } } return idxFile, idxStat.Size(), nil @@ -50,7 +61,7 @@ func (v *Volume) ScrubIndex() (int64, []error) { } // scrubVolumeData checks a volume content + index for issues. -func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *os.File, idxFileSize int64) (int64, []error) { +func (v *Volume) scrubVolumeData(idxFile *os.File, idxFileSize int64) (int64, []error) { // full scrubbing means also scrubbing the index var count int64 _, errs := idx.CheckIndexFile(idxFile, idxFileSize, v.Version()) @@ -73,8 +84,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o } n := needle.Needle{} - if err := n.ReadData(dataFile, offset.ToActualOffset(), size, version); err != nil { - errs = append(errs, fmt.Errorf("needle %d on volume %d: %v", id, v.Id, err)) + if err := n.ReadData(v.DataBackend, offset.ToActualOffset(), size, version); err != nil { + errs = append(errs, fmt.Errorf("failed to read needle %d on volume %d: %v", id, v.Id, err)) } totalRead += actualSize @@ -86,7 +97,8 @@ func (v *Volume) scrubVolumeData(dataFile backend.BackendStorageFile, idxFile *o // check total volume file size wantSize := totalRead + super_block.SuperBlockSize - dataSize, _, err := dataFile.GetStat() + dataSize, _, err := v.DataBackend.GetStat() + if err != nil { errs = append(errs, fmt.Errorf("failed to stat data file for volume %d: %v", v.Id, err)) } else { @@ -111,7 +123,7 @@ func (v *Volume) Scrub() (int64, []error) { } defer idxFile.Close() - return v.scrubVolumeData(v.DataBackend, idxFile, idxFileSize) + return v.scrubVolumeData(idxFile, idxFileSize) } func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uint64, err error) { @@ -175,7 +187,6 @@ func doCheckAndFixVolumeData(v *Volume, indexFile *os.File, indexOffset int64) ( return lastAppendAtNs, nil } - func verifyIndexFileIntegrity(indexFile *os.File) (indexSize int64, err error) { if indexSize, err = util.GetFileSize(indexFile); err == nil { if indexSize%types.NeedleMapEntrySize != 0 { diff --git a/weed/storage/volume_checking_test.go b/weed/storage/volume_checking_test.go index 2a57f36769..6ba82df686 100644 --- a/weed/storage/volume_checking_test.go +++ b/weed/storage/volume_checking_test.go @@ -30,6 +30,51 @@ func TestScrubVolumeData(t *testing.T) { want: 27, wantErrs: []error{}, }, + { + name: "zero-size volume without index", + dataPath: "./test_files/empty_volume.dat", + indexPath: "./test_files/empty_volume.idx", + version: needle.Version3, + want: 0, + wantErrs: []error{}, + }, + { + name: "zero-size volume with index", + dataPath: "./test_files/empty_volume.dat", + indexPath: "./test_files/healthy_volume.idx", + version: needle.Version3, + want: 27, + wantErrs: []error{ + fmt.Errorf("failed to read needle 3 on volume 0: EOF"), + fmt.Errorf("failed to read needle 4 on volume 0: EOF"), + fmt.Errorf("failed to read needle 8 on volume 0: EOF"), + fmt.Errorf("failed to read needle 9 on volume 0: EOF"), + fmt.Errorf("failed to read needle 11 on volume 0: EOF"), + fmt.Errorf("failed to read needle 18 on volume 0: EOF"), + fmt.Errorf("failed to read needle 20 on volume 0: EOF"), + fmt.Errorf("failed to read needle 21 on volume 0: EOF"), + fmt.Errorf("failed to read needle 25 on volume 0: EOF"), + fmt.Errorf("failed to read needle 29 on volume 0: EOF"), + fmt.Errorf("failed to read needle 31 on volume 0: EOF"), + fmt.Errorf("failed to read needle 33 on volume 0: EOF"), + fmt.Errorf("failed to read needle 35 on volume 0: EOF"), + fmt.Errorf("failed to read needle 39 on volume 0: EOF"), + fmt.Errorf("failed to read needle 45 on volume 0: EOF"), + fmt.Errorf("failed to read needle 46 on volume 0: EOF"), + fmt.Errorf("failed to read needle 48 on volume 0: EOF"), + fmt.Errorf("data file for volume 0 is smaller (8) than the 27 needles it contains (942856)"), + }, + }, + { + name: "volume without index", + dataPath: "./test_files/healthy_volume.dat", + indexPath: "./test_files/empty_volume.idx", + version: needle.Version3, + want: 0, + wantErrs: []error{ + fmt.Errorf("data file size for volume 0 (942856) doesn't match the size for 0 needles read (8)"), + }, + }, { name: "bitrot volume", dataPath: "./test_files/bitrot_volume.dat", @@ -37,8 +82,8 @@ func TestScrubVolumeData(t *testing.T) { version: needle.Version3, want: 27, wantErrs: []error{ - fmt.Errorf("needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"), - fmt.Errorf("needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"), + fmt.Errorf("failed to read needle 3 on volume 0: invalid CRC for needle 3 (got 0b243a0d, want 4af853fb), data on disk corrupted: needle data corrupted"), + fmt.Errorf("failed to read needle 48 on volume 0: invalid CRC for needle 30 (got 3c40e8d5, want 5077fea1), data on disk corrupted: needle data corrupted"), fmt.Errorf("data file size for volume 0 (942864) doesn't match the size for 27 needles read (942856)"), }, }, @@ -64,12 +109,13 @@ func TestScrubVolumeData(t *testing.T) { } v := Volume{ + DataBackend: backend.NewDiskFile(datFile), volumeInfo: &volume_server_pb.VolumeInfo{ Version: uint32(tc.version), }, } - got, gotErrs := v.scrubVolumeData(backend.NewDiskFile(datFile), idxFile, idxStat.Size()) + got, gotErrs := v.scrubVolumeData(idxFile, idxStat.Size()) if got != tc.want { t.Errorf("expected %d files processed, got %d", tc.want, got)