volume: check the needle id in the streamed-read probe and on blob writes (#12794)

* storage: reject a streamed read whose record id or size differs

The header probe before a streamed read returned not-found only when both
the record's id and its size differed from the index entry, so a record of
another needle with the same size passed and its data was streamed under
the requested key until the final checksum check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: refuse a needle blob whose header names another needle

WriteNeedleBlob compared only the blob header's size with the size it was
asked to index. A blob read from an offset that has since moved to another
record of the same size was appended and indexed under the wrong key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): refuse a needle blob whose header names another needle

Mirror of the Go check: write_needle_blob_and_index compared only the blob
header's size, so a blob of another needle with the same size was appended
and indexed under the requested key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): report not-found when a streamed read's record is another needle

The meta read ahead of a streamed reply took the id from the record header
without comparing it with the requested key. Return not-found when the
index entry resolves to another needle, as the Go header probe does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: check the requested id after a meta-only read

ReadNeedleMeta overwrites n.Id with the record's header id, so a meta-only
read through an index entry that resolves to another needle returned that
needle's metadata, and the streamed-read probe that follows compared the
record with itself. Return not-found when the header names another needle,
as the Rust meta read does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: copy a needle blob to another volume, not under another key

WriteNeedleBlob now refuses a blob whose header names another needle,
so the round trip writes the blob under its own id into a second volume
and checks that storing it under another key is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: compare the streamed header with a deleted entry's record size

A deleted index entry stores the negated size, so a read-deleted stream
compared the positive header size with a negative one and missed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chris LuandClaude Opus 5.5 authored and GitHub committed 2026-11-19 15:27:15 -08:00
1 parent 4a5dbe4c6f
commit 4ac3633f1c
6 files changed
+209 -26

No files matched your search

+64 -2
View File
@@ -897,6 +897,13 @@ impl NeedleReadPlan {
self.io_errors.check_read_write_error(Some(e));
}
result?;
if n.id != self.needle_id {
warn!(
"index entry for needle {} resolves to needle {}; reporting not-found",
self.needle_id.0, n.id.0
);
return Err(VolumeError::NotFound);
}
if needle_expired(n) {
return Err(VolumeError::NotFound);
}
@@ -5666,7 +5673,7 @@ impl Volume {
),
)));
}
let (_, _, header_size) = Needle::parse_header(needle_blob);
let (_, header_id, header_size) = Needle::parse_header(needle_blob);
if header_size != size {
return Err(VolumeError::Io(io::Error::new(
io::ErrorKind::InvalidData,
@@ -5676,6 +5683,15 @@ impl Volume {
),
)));
}
if header_id != needle_id {
return Err(VolumeError::Io(io::Error::new(
io::ErrorKind::InvalidData,
format!(
"needle {} blob header holds needle {}",
needle_id.0, header_id.0
),
)));
}
// The blob is appended as is: its length must be the record this size takes in this volume's version.
let actual_size = get_actual_size(size, self.version());
if needle_blob.len() as i64 != actual_size {
@@ -10731,6 +10747,15 @@ mod tests {
assert!(matches!(err, VolumeError::Io(_)), "got {err:?}");
assert_eq!(v.dat_file_size().unwrap(), dat_size_before);
// needle 1's blob is not stored under key 2
let err = v
.write_needle_blob_and_index(NeedleId(2), &blob, n.size)
.unwrap_err();
assert!(matches!(err, VolumeError::Io(_)), "got {err:?}");
assert_eq!(v.dat_file_size().unwrap(), dat_size_before);
let mut blob = blob;
NeedleId(2).to_bytes(&mut blob[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
v.write_needle_blob_and_index(NeedleId(2), &blob, n.size)
.unwrap();
}
@@ -10753,7 +10778,8 @@ mod tests {
};
n.checksum = CRC::new(&n.data);
let (offset, _, _) = v.write_needle(&mut n, true, false).unwrap();
let blob = v.read_needle_blob(offset as i64, n.size).unwrap();
let mut blob = v.read_needle_blob(offset as i64, n.size).unwrap();
NeedleId(2).to_bytes(&mut blob[COOKIE_SIZE..COOKIE_SIZE + NEEDLE_ID_SIZE]);
let dat_size_before = v.dat_file_size().unwrap();
@@ -11223,6 +11249,42 @@ mod tests {
);
}
#[test]
fn test_read_meta_rejects_index_entry_naming_another_needle() {
let tmp = TempDir::new().unwrap();
let dir = tmp.path().to_str().unwrap();
let v = make_test_volume(dir);
for id in [1u64, 2] {
let mut n = Needle {
id: NeedleId(id),
cookie: Cookie(0xDEADBEEF),
data: vec![id as u8; 2048],
data_size: 2048,
..Needle::default()
};
v.write_needle(&mut n, true, false).unwrap();
}
// point key 9 at needle 2's record with the record's own size
{
let mut io = v.io.write().unwrap();
let nm = io.nm.as_mut().unwrap();
let nv = nm.get(NeedleId(2)).unwrap().unwrap();
nm.put(NeedleId(9), nv.offset, nv.size).unwrap();
}
let mut read_n = Needle {
id: NeedleId(9),
cookie: Cookie(0xDEADBEEF),
..Needle::default()
};
let mut plan = v.needle_read_plan(NeedleId(9), false).unwrap();
assert!(matches!(
plan.read_meta(&mut read_n),
Err(VolumeError::NotFound)
));
}
#[test]
fn test_stream_info_locates_needle_data() {
let tmp = TempDir::new().unwrap();
@@ -28,7 +28,8 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
const volumeID = uint32(83)
const sourceNeedleID = uint64(333333)
const sourceCookie = uint32(0xABCD0102)
const clonedNeedleID = uint64(333334)
const otherNeedleID = uint64(333334)
const copyVolumeID = uint32(183)
framework.AllocateVolume(t, grpcClient, volumeID, "")
@@ -87,7 +88,24 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
VolumeId: volumeID,
NeedleId: clonedNeedleID,
NeedleId: otherNeedleID,
Size: size,
NeedleBlob: blobResp.GetNeedleBlob(),
})
if err == nil || !strings.Contains(err.Error(), "blob header holds needle") {
t.Fatalf("WriteNeedleBlob under another needle id should be refused, got %v", err)
}
if _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: otherNeedleID,
}); err == nil {
t.Fatalf("refused blob should not be indexed under needle %d", otherNeedleID)
}
framework.AllocateVolume(t, grpcClient, copyVolumeID, "")
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
VolumeId: copyVolumeID,
NeedleId: sourceNeedleID,
Size: size,
NeedleBlob: blobResp.GetNeedleBlob(),
})
@@ -95,27 +113,27 @@ func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
t.Fatalf("WriteNeedleBlob failed: %v", err)
}
clonedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: clonedNeedleID,
copiedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: copyVolumeID,
NeedleId: sourceNeedleID,
})
if err != nil {
t.Fatalf("VolumeNeedleStatus for cloned needle failed: %v", err)
t.Fatalf("VolumeNeedleStatus for copied needle failed: %v", err)
}
if clonedStatus.GetNeedleId() != sourceNeedleID {
t.Fatalf("cloned needle status id mismatch: got %d want %d", clonedStatus.GetNeedleId(), sourceNeedleID)
if copiedStatus.GetNeedleId() != sourceNeedleID {
t.Fatalf("copied needle status id mismatch: got %d want %d", copiedStatus.GetNeedleId(), sourceNeedleID)
}
if clonedStatus.GetCookie() != sourceCookie {
t.Fatalf("cloned needle cookie mismatch: got %d want %d", clonedStatus.GetCookie(), sourceCookie)
if copiedStatus.GetCookie() != sourceCookie {
t.Fatalf("copied needle cookie mismatch: got %d want %d", copiedStatus.GetCookie(), sourceCookie)
}
clonedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, clonedNeedleID, sourceCookie))
clonedReadBody := framework.ReadAllAndClose(t, clonedReadResp)
if clonedReadResp.StatusCode != 200 {
t.Fatalf("cloned needle GET expected 200, got %d", clonedReadResp.StatusCode)
copiedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(copyVolumeID, sourceNeedleID, sourceCookie))
copiedReadBody := framework.ReadAllAndClose(t, copiedReadResp)
if copiedReadResp.StatusCode != 200 {
t.Fatalf("copied needle GET expected 200, got %d", copiedReadResp.StatusCode)
}
if string(clonedReadBody) != string(payload) {
t.Fatalf("cloned needle body mismatch: got %q want %q", string(clonedReadBody), string(payload))
if string(copiedReadBody) != string(payload) {
t.Fatalf("copied needle body mismatch: got %q want %q", string(copiedReadBody), string(payload))
}
}
+54
View File
@@ -222,6 +222,60 @@ func TestRepairIdxHeadTombstones_KeepsDeletedNeedlesDeleted(t *testing.T) {
}
}
// TestStaleIndexEntry_SameSizeStreamMisses covers a damaged index entry that
// resolves to another needle of the same size.
func TestStaleIndexEntry_SameSizeStreamMisses(t *testing.T) {
dir := t.TempDir()
written := writeTestVolume(t, dir, 5)
v := mustReload(t, dir)
defer v.Close()
const staleKey = 9
nv, ok := v.nm.Get(types.Uint64ToNeedleId(2))
if !ok {
t.Fatal("needle 2 missing from index")
}
if err := v.nm.Put(types.Uint64ToNeedleId(staleKey), nv.Offset, nv.Size); err != nil {
t.Fatalf("plant stale index entry: %v", err)
}
meta := newEmptyNeedle(staleKey)
if _, err := v.readNeedle(meta, &ReadOption{AttemptMetaOnly: true, MustMetaOnly: true}, nil); err != ErrorNotFound {
t.Fatalf("stale-index meta read: want ErrorNotFound, got %v (needle %x)", err, uint64(meta.Id))
}
stream := newEmptyNeedle(staleKey)
stream.DataSize = written[2].DataSize
var out bytes.Buffer
if err := v.readNeedleDataInto(stream, &ReadOption{ReadBufferSize: 1024}, &out, 0, int64(stream.DataSize)); err != ErrorNotFound {
t.Fatalf("stale-index stream: want ErrorNotFound, got %v after %d bytes", err, out.Len())
}
}
func TestDeletedNeedleStreamsWithReadDeleted(t *testing.T) {
dir := t.TempDir()
written := writeTestVolume(t, dir, 3)
v := mustReload(t, dir)
defer v.Close()
if _, err := v.deleteNeedle2(newEmptyNeedle(2), false); err != nil {
t.Fatalf("delete needle 2: %v", err)
}
stream := newEmptyNeedle(2)
stream.DataSize = written[2].DataSize
stream.Checksum = written[2].Checksum
var out bytes.Buffer
if err := v.readNeedleDataInto(stream, &ReadOption{ReadDeleted: true, ReadBufferSize: 1024}, &out, 0, int64(stream.DataSize)); err != nil {
t.Fatalf("stream deleted needle: %v", err)
}
if !bytes.Equal(out.Bytes(), written[2].Data) {
t.Fatalf("streamed %d bytes, want the %d bytes written", out.Len(), len(written[2].Data))
}
}
// TestStaleIndexEntry_HealsOnWrite covers a damaged index entry that resolves
// to a different needle: reads must miss instead of serving the wrong needle,
// and the next write for the key must re-index it rather than be rejected as
+11 -1
View File
@@ -94,6 +94,7 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
// data, which ReadNeedleMeta relies on, so it is always read in full.
if readOption != nil && readOption.AttemptMetaOnly && v.Version() != needle.Version1 && (readOption.MustMetaOnly || readOption.HasRange || readSize > PagedReadLimit) {
readOption.VolumeRevision = v.SuperBlock.CompactionRevision
id := n.Id
err = n.ReadNeedleMeta(v.DataBackend, nv.Offset.ToActualOffset(), readSize, v.Version())
if err == needle.ErrorSizeMismatch && OffsetSize == 4 {
readOption.IsOutOfRange = true
@@ -103,6 +104,11 @@ func (v *Volume) readNeedle(n *needle.Needle, readOption *ReadOption, onReadSize
v.checkReadWriteError(err)
return 0, err
}
if n.Id != id {
glog.V(0).Infof("volume %d index entry for needle %x resolves to needle %x; reporting not-found",
v.Id, uint64(id), uint64(n.Id))
return -1, ErrorNotFound
}
if !n.IsCompressed() && !n.IsChunkedManifest() {
readOption.IsMetaOnly = true
readOption.metaOffset = nv.Offset
@@ -438,7 +444,11 @@ func (v *Volume) streamNeedleData(n *needle.Needle, readOption *ReadOption, writ
}
probe := new(needle.Needle)
probe.ParseNeedleHeader(header)
if probe.Id != n.Id && probe.Size != nv.Size {
recordSize := nv.Size
if recordSize < 0 {
recordSize = -recordSize
}
if probe.Id != n.Id || probe.Size != recordSize {
glog.V(0).Infof("volume %d index entry for needle %x resolves to needle %x; reporting not-found",
v.Id, uint64(n.Id), uint64(probe.Id))
return ErrorNotFound
+3
View File
@@ -907,6 +907,9 @@ func (v *Volume) WriteNeedleBlob(needleId NeedleId, needleBlob []byte, size Size
if blobHeader.Size != size {
return fmt.Errorf("needle %d size %d does not match its blob header size %d", needleId, size, blobHeader.Size)
}
if blobHeader.Id != needleId {
return fmt.Errorf("needle %d blob header holds needle %d", needleId, blobHeader.Id)
}
// The blob is appended as is: its length must be the record this size takes in this volume's version.
if actualSize := needle.GetActualSize(size, v.Version()); int64(len(needleBlob)) != actualSize {
return fmt.Errorf("needle %d blob of %d bytes does not match the %d bytes size %d takes in a version %d volume", needleId, len(needleBlob), actualSize, size, v.Version())
+43 -7
View File
@@ -207,7 +207,7 @@ func TestWriteNeedleBlobRejectedOnReadOnlyVolume(t *testing.T) {
datSizeBefore, _, _ := v.DataBackend.GetStat()
err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, n.Size)
err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), withNeedleId(blob, 2), n.Size)
if err == nil {
t.Fatalf("expected WriteNeedleBlob to be rejected on a read-only volume")
}
@@ -249,7 +249,7 @@ func TestWriteNeedleBlobRejectsSizeMismatch(t *testing.T) {
// types.Size(n.DataSize) is what needle.Append reports, and what a caller
// following the payload-size convention would send.
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, types.Size(n.DataSize)); err == nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), withNeedleId(blob, 2), types.Size(n.DataSize)); err == nil {
t.Fatal("expected WriteNeedleBlob to reject a size that disagrees with the blob header")
}
@@ -258,7 +258,7 @@ func TestWriteNeedleBlobRejectsSizeMismatch(t *testing.T) {
t.Errorf(".dat grew from %d to %d on a rejected blob", datSizeBefore, datSizeAfter)
}
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, n.Size); err != nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), withNeedleId(blob, 2), n.Size); err != nil {
t.Fatalf("write needle blob with the header size: %v", err)
}
}
@@ -325,7 +325,7 @@ func TestWriteNeedleBlobRejectsNegativeSize(t *testing.T) {
datSizeBefore, _, _ := v.DataBackend.GetStat()
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, size); err == nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), withNeedleId(blob, 2), size); err == nil {
t.Error("expected WriteNeedleBlob to reject a negative size")
}
@@ -371,7 +371,7 @@ func TestWriteNeedleBlobRejectsLengthMismatch(t *testing.T) {
}
datSizeBefore, _, _ := v.DataBackend.GetStat()
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), tc.mutate(blob), n.Size); err == nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), tc.mutate(withNeedleId(blob, 2)), n.Size); err == nil {
t.Error("expected WriteNeedleBlob to reject a blob whose length does not match its size")
}
@@ -414,7 +414,7 @@ func TestWriteNeedleBlobRoundTrip(t *testing.T) {
if err != nil {
t.Fatalf("read needle blob: %v", err)
}
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, n.Size); err != nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), withNeedleId(blob, 2), n.Size); err != nil {
t.Fatalf("write needle blob: %v", err)
}
got := newEmptyNeedle(2)
@@ -433,7 +433,7 @@ func TestWriteNeedleBlobRoundTrip(t *testing.T) {
if err != nil {
t.Fatalf("read delete record: %v", err)
}
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(3), deleteRecord, 0); err != nil {
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(3), withNeedleId(deleteRecord, 3), 0); err != nil {
t.Fatalf("write size-0 needle blob: %v", err)
}
@@ -581,3 +581,39 @@ func TestSyncBatchDefersFlushForStragglers(t *testing.T) {
t.Fatalf("expected a lone write to take its own sync, got %d total", syncs)
}
}
// withNeedleId returns a copy of blob whose header names needle id.
func withNeedleId(blob []byte, id uint64) []byte {
out := append([]byte(nil), blob...)
types.NeedleIdToBytes(out[types.CookieSize:types.CookieSize+types.NeedleIdSize], types.Uint64ToNeedleId(id))
return out
}
func TestWriteNeedleBlobRejectsAnotherNeedlesBlob(t *testing.T) {
dir := t.TempDir()
v, err := NewVolume(dir, dir, "", 7, NeedleMapInMemory, &super_block.ReplicaPlacement{}, &needle.TTL{}, 0, needle.GetCurrentVersion(), 0, 0)
if err != nil {
t.Fatalf("volume creation: %v", err)
}
defer v.Close()
n := newRandomNeedle(1)
offset, _, _, err := v.writeNeedle2(n, true, false, false)
if err != nil {
t.Fatalf("write needle: %v", err)
}
blob, err := v.ReadNeedleBlob(int64(offset), n.Size)
if err != nil {
t.Fatalf("read needle blob: %v", err)
}
datSizeBefore, _, _ := v.DataBackend.GetStat()
if err = v.WriteNeedleBlob(types.Uint64ToNeedleId(2), blob, n.Size); err == nil {
t.Fatal("needle 1's blob was stored under key 2")
}
if datSizeAfter, _, _ := v.DataBackend.GetStat(); datSizeAfter != datSizeBefore {
t.Errorf("rejected blob grew .dat from %d to %d bytes", datSizeBefore, datSizeAfter)
}
if _, ok := v.nm.Get(types.Uint64ToNeedleId(2)); ok {
t.Error("rejected blob was indexed under key 2")
}
}