Files
seaweedfs/test/volume_server/grpc/data_stream_success_test.go
T
Chris LuandClaude Opus 5.5 4ac3633f1c volume: check the needle id in the streamed-read probe and on blob writes (#12794)
* storage: reject a streamed read whose record id or size differs

The header probe before a streamed read returned not-found only when both
the record's id and its size differed from the index entry, so a record of
another needle with the same size passed and its data was streamed under
the requested key until the final checksum check.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: refuse a needle blob whose header names another needle

WriteNeedleBlob compared only the blob header's size with the size it was
asked to index. A blob read from an offset that has since moved to another
record of the same size was appended and indexed under the wrong key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): refuse a needle blob whose header names another needle

Mirror of the Go check: write_needle_blob_and_index compared only the blob
header's size, so a blob of another needle with the same size was appended
and indexed under the requested key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* volume(rs): report not-found when a streamed read's record is another needle

The meta read ahead of a streamed reply took the id from the record header
without comparing it with the requested key. Return not-found when the
index entry resolves to another needle, as the Go header probe does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: check the requested id after a meta-only read

ReadNeedleMeta overwrites n.Id with the record's header id, so a meta-only
read through an index entry that resolves to another needle returned that
needle's metadata, and the streamed-read probe that follows compared the
record with itself. Return not-found when the header names another needle,
as the Rust meta read does.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* test: copy a needle blob to another volume, not under another key

WriteNeedleBlob now refuses a blob whose header names another needle,
so the round trip writes the blob under its own id into a second volume
and checks that storing it under another key is refused.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* storage: compare the streamed header with a deleted entry's record size

A deleted index entry stores the negated size, so a read-deleted stream
compared the positive header size with a negative one and missed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-19 15:27:15 -08:00

447 lines
14 KiB
Go

package volume_server_grpc_test
import (
"context"
"io"
"net/http"
"reflect"
"strings"
"testing"
"time"
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
"github.com/seaweedfs/seaweedfs/weed/storage/idx"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func TestReadWriteNeedleBlobAndMetaRoundTrip(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(83)
const sourceNeedleID = uint64(333333)
const sourceCookie = uint32(0xABCD0102)
const otherNeedleID = uint64(333334)
const copyVolumeID = uint32(183)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
payload := []byte("blob-roundtrip-content")
fid := framework.NewFileID(volumeID, sourceNeedleID, sourceCookie)
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != 201 {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("ReadVolumeFileStatus failed: %v", err)
}
if fileStatus.GetIdxFileSize() == 0 {
t.Fatalf("expected non-zero idx file size after upload")
}
idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".idx",
CompactionRevision: fileStatus.GetCompactionRevision(),
StopOffset: fileStatus.GetIdxFileSize(),
})
offset, size := findNeedleOffsetAndSize(t, idxBytes, sourceNeedleID)
blobResp, err := grpcClient.ReadNeedleBlob(ctx, &volume_server_pb.ReadNeedleBlobRequest{
VolumeId: volumeID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleBlob failed: %v", err)
}
if len(blobResp.GetNeedleBlob()) == 0 {
t.Fatalf("ReadNeedleBlob returned empty blob")
}
metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
VolumeId: volumeID,
NeedleId: sourceNeedleID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleMeta failed: %v", err)
}
if metaResp.GetCookie() != sourceCookie {
t.Fatalf("ReadNeedleMeta cookie mismatch: got %d want %d", metaResp.GetCookie(), sourceCookie)
}
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
VolumeId: volumeID,
NeedleId: otherNeedleID,
Size: size,
NeedleBlob: blobResp.GetNeedleBlob(),
})
if err == nil || !strings.Contains(err.Error(), "blob header holds needle") {
t.Fatalf("WriteNeedleBlob under another needle id should be refused, got %v", err)
}
if _, err = grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: volumeID,
NeedleId: otherNeedleID,
}); err == nil {
t.Fatalf("refused blob should not be indexed under needle %d", otherNeedleID)
}
framework.AllocateVolume(t, grpcClient, copyVolumeID, "")
_, err = grpcClient.WriteNeedleBlob(ctx, &volume_server_pb.WriteNeedleBlobRequest{
VolumeId: copyVolumeID,
NeedleId: sourceNeedleID,
Size: size,
NeedleBlob: blobResp.GetNeedleBlob(),
})
if err != nil {
t.Fatalf("WriteNeedleBlob failed: %v", err)
}
copiedStatus, err := grpcClient.VolumeNeedleStatus(ctx, &volume_server_pb.VolumeNeedleStatusRequest{
VolumeId: copyVolumeID,
NeedleId: sourceNeedleID,
})
if err != nil {
t.Fatalf("VolumeNeedleStatus for copied needle failed: %v", err)
}
if copiedStatus.GetNeedleId() != sourceNeedleID {
t.Fatalf("copied needle status id mismatch: got %d want %d", copiedStatus.GetNeedleId(), sourceNeedleID)
}
if copiedStatus.GetCookie() != sourceCookie {
t.Fatalf("copied needle cookie mismatch: got %d want %d", copiedStatus.GetCookie(), sourceCookie)
}
copiedReadResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(copyVolumeID, sourceNeedleID, sourceCookie))
copiedReadBody := framework.ReadAllAndClose(t, copiedReadResp)
if copiedReadResp.StatusCode != 200 {
t.Fatalf("copied needle GET expected 200, got %d", copiedReadResp.StatusCode)
}
if string(copiedReadBody) != string(payload) {
t.Fatalf("copied needle body mismatch: got %q want %q", string(copiedReadBody), string(payload))
}
}
func TestReadAllNeedlesStreamsUploadedRecords(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(84)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
expected := map[uint64]string{
444441: "read-all-needle-one",
444442: "read-all-needle-two",
}
for key, body := range expected {
resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), framework.NewFileID(volumeID, key, 0xA0B0C0D0), []byte(body))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != 201 {
t.Fatalf("upload for key %d expected 201, got %d", key, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
t.Fatalf("ReadAllNeedles start failed: %v", err)
}
seen := map[uint64]string{}
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
}
if _, wanted := expected[msg.GetNeedleId()]; wanted {
seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
}
}
for key, body := range expected {
got, found := seen[key]
if !found {
t.Fatalf("ReadAllNeedles missing key %d in stream", key)
}
if got != body {
t.Fatalf("ReadAllNeedles body mismatch for key %d: got %q want %q", key, got, body)
}
}
}
func TestReadAllNeedlesExistingThenMissingVolumeAbortsStream(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const existingVolumeID = uint32(85)
const missingVolumeID = uint32(98585)
const needleID = uint64(445551)
framework.AllocateVolume(t, grpcClient, existingVolumeID, "")
client := framework.NewHTTPClient()
fid := framework.NewFileID(existingVolumeID, needleID, 0xAA11BB22)
payload := "read-all-existing-then-missing"
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte(payload))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != 201 {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
VolumeIds: []uint32{existingVolumeID, missingVolumeID},
})
if err != nil {
t.Fatalf("ReadAllNeedles start failed: %v", err)
}
seenUploadedNeedle := false
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
t.Fatalf("ReadAllNeedles expected stream error for missing volume, got EOF")
}
if recvErr != nil {
if !strings.Contains(recvErr.Error(), "not found volume id") {
t.Fatalf("ReadAllNeedles missing-volume error mismatch: %v", recvErr)
}
break
}
if msg.GetNeedleId() == needleID && string(msg.GetNeedleBlob()) == payload {
seenUploadedNeedle = true
}
}
if !seenUploadedNeedle {
t.Fatalf("ReadAllNeedles should stream entries from existing volume before missing-volume abort")
}
}
func TestReadAllNeedlesPreservesDatOrderAcrossOverwrite(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(86)
const firstNeedleID = uint64(444551)
const secondNeedleID = uint64(444552)
const firstCookie = uint32(0xAA22BB33)
const secondCookie = uint32(0xCC44DD55)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
uploads := []struct {
fid string
body string
}{
{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first"},
{fid: framework.NewFileID(volumeID, secondNeedleID, secondCookie), body: "read-all-second"},
{fid: framework.NewFileID(volumeID, firstNeedleID, firstCookie), body: "read-all-first-overwrite"},
}
for _, upload := range uploads {
resp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), upload.fid, []byte(upload.body))
_ = framework.ReadAllAndClose(t, resp)
if resp.StatusCode != 201 {
t.Fatalf("upload for %s expected 201, got %d", upload.fid, resp.StatusCode)
}
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{VolumeIds: []uint32{volumeID}})
if err != nil {
t.Fatalf("ReadAllNeedles start failed: %v", err)
}
var orderedIDs []uint64
var orderedBodies []string
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
break
}
if recvErr != nil {
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
}
orderedIDs = append(orderedIDs, msg.GetNeedleId())
orderedBodies = append(orderedBodies, string(msg.GetNeedleBlob()))
}
wantIDs := []uint64{secondNeedleID, firstNeedleID}
wantBodies := []string{"read-all-second", "read-all-first-overwrite"}
if !reflect.DeepEqual(orderedIDs, wantIDs) {
t.Fatalf("ReadAllNeedles order mismatch: got %v want %v", orderedIDs, wantIDs)
}
if !reflect.DeepEqual(orderedBodies, wantBodies) {
t.Fatalf("ReadAllNeedles bodies mismatch: got %v want %v", orderedBodies, wantBodies)
}
}
func TestReadNeedleMetaDeletedEntryUsesTombstoneMetadata(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test in short mode")
}
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
defer conn.Close()
const volumeID = uint32(87)
const needleID = uint64(444661)
const cookie = uint32(0xAB12CD34)
framework.AllocateVolume(t, grpcClient, volumeID, "")
client := framework.NewHTTPClient()
fid := framework.NewFileID(volumeID, needleID, cookie)
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("read-meta-delete"))
_ = framework.ReadAllAndClose(t, uploadResp)
if uploadResp.StatusCode != http.StatusCreated {
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
}
deleteReq, err := http.NewRequest(http.MethodDelete, clusterHarness.VolumeAdminURL()+"/"+fid, nil)
if err != nil {
t.Fatalf("build delete request: %v", err)
}
deleteResp := framework.DoRequest(t, client, deleteReq)
_ = framework.ReadAllAndClose(t, deleteResp)
if deleteResp.StatusCode != http.StatusAccepted {
t.Fatalf("delete expected 202, got %d", deleteResp.StatusCode)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
fileStatus, err := grpcClient.ReadVolumeFileStatus(ctx, &volume_server_pb.ReadVolumeFileStatusRequest{VolumeId: volumeID})
if err != nil {
t.Fatalf("ReadVolumeFileStatus after delete failed: %v", err)
}
idxBytes := copyFileBytes(t, grpcClient, &volume_server_pb.CopyFileRequest{
VolumeId: volumeID,
Ext: ".idx",
CompactionRevision: fileStatus.GetCompactionRevision(),
StopOffset: fileStatus.GetIdxFileSize(),
})
offset, size := findLastNeedleOffsetAndSize(t, idxBytes, needleID)
if size >= 0 {
t.Fatalf("expected deleted idx entry for needle %d, got size %d", needleID, size)
}
metaResp, err := grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
VolumeId: volumeID,
NeedleId: needleID,
Offset: offset,
Size: size,
})
if err != nil {
t.Fatalf("ReadNeedleMeta deleted-entry failed: %v", err)
}
if metaResp.GetCookie() != cookie {
t.Fatalf("ReadNeedleMeta deleted-entry cookie mismatch: got %d want %d", metaResp.GetCookie(), cookie)
}
if metaResp.GetAppendAtNs() == 0 {
t.Fatalf("ReadNeedleMeta deleted-entry expected non-zero append_at_ns")
}
}
func copyFileBytes(t testing.TB, grpcClient volume_server_pb.VolumeServerClient, req *volume_server_pb.CopyFileRequest) []byte {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
stream, err := grpcClient.CopyFile(ctx, req)
if err != nil {
t.Fatalf("CopyFile start failed: %v", err)
}
var out []byte
for {
msg, recvErr := stream.Recv()
if recvErr == io.EOF {
return out
}
if recvErr != nil {
t.Fatalf("CopyFile recv failed: %v", recvErr)
}
out = append(out, msg.GetFileContent()...)
}
}
func findNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
if entryOffset.IsZero() || entrySize <= 0 {
continue
}
return entryOffset.ToActualOffset(), int32(entrySize)
}
t.Fatalf("needle id %d not found in idx entries", needleID)
return 0, 0
}
func findLastNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64) (offset int64, size int32) {
t.Helper()
found := false
for i := 0; i+types.NeedleMapEntrySize <= len(idxBytes); i += types.NeedleMapEntrySize {
key, entryOffset, entrySize := idx.IdxFileEntry(idxBytes[i : i+types.NeedleMapEntrySize])
if uint64(key) != needleID {
continue
}
found = true
offset = entryOffset.ToActualOffset()
size = int32(entrySize)
}
if !found {
t.Fatalf("needle id %d not found in idx entries", needleID)
}
return offset, size
}