mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-10-31 09:20:50 +01:00
filer: keep ciphered chunks on the whole-chunk path
A range fetch cannot save bytes for a ciphered chunk: readEncryptedUrl always downloads and decrypts the whole blob before slicing. Sending partial views of ciphered chunks through fetchChunkRange would repeat the full download per buffer, so they keep the shared whole-chunk path where one download serves every buffer. Prefetch stays enabled for them for the same reason.
This commit is contained in:
1 parent
dd75add62f
commit
521a19c567
3 files changed
+27
-10
No files matched your search
@@ -352,8 +352,11 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
|
||||
// A view clipped to part of its chunk (e.g. the edge of a ranged GET,
|
||||
// whose views ViewFromVisibleIntervals clips to the request) only ever
|
||||
// needs that part: fetch it as a range no matter the detected pattern.
|
||||
// Fetching the chunk whole would multiply volume-server reads.
|
||||
if !chunkView.IsFullChunk() || c.readerPattern.IsRandomMode() {
|
||||
// Fetching the chunk whole would multiply volume-server reads. Ciphered
|
||||
// chunks are the exception: the wire fetch is always the whole encrypted
|
||||
// blob, so they take the shared whole-chunk path where one download
|
||||
// serves every buffer.
|
||||
if chunkView.CipherKey == nil && (!chunkView.IsFullChunk() || c.readerPattern.IsRandomMode()) {
|
||||
c.readerCache.releaseStream(&c.stream)
|
||||
n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
|
||||
if n > 0 {
|
||||
@@ -383,7 +386,7 @@ func (c *ChunkReadAt) readChunkSliceAt(ctx context.Context, buffer []byte, chunk
|
||||
// readChunkSliceAtForParallel is a simplified version for parallel chunk fetching
|
||||
// It doesn't update lastChunkFid or trigger prefetch (handled by the caller)
|
||||
func (c *ChunkReadAt) readChunkSliceAtForParallel(ctx context.Context, buffer []byte, chunkView *ChunkView, offset uint64) (n int, err error) {
|
||||
if !chunkView.IsFullChunk() {
|
||||
if chunkView.CipherKey == nil && !chunkView.IsFullChunk() {
|
||||
n, err = c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
|
||||
if n > 0 {
|
||||
return n, err
|
||||
|
||||
@@ -248,7 +248,8 @@ func TestChunkReadAtClippedViewsFetchOnlyCoveredParts(t *testing.T) {
|
||||
defer rc.destroy()
|
||||
fetches := fetchRecorder(rc)
|
||||
|
||||
// Window [56KiB, 136KiB): tail of chunk0, all of chunk1, head of chunk2.
|
||||
// Window [56KiB, 144KiB): tail of chunk0, all of chunk1, head of
|
||||
// chunk2, head of ciphered chunk3 (file chunks need not be aligned).
|
||||
views := NewIntervalList[*ChunkView]()
|
||||
views.AppendInterval(&Interval[*ChunkView]{
|
||||
StartOffset: chunkSize - 8<<10,
|
||||
@@ -265,17 +266,25 @@ func TestChunkReadAtClippedViewsFetchOnlyCoveredParts(t *testing.T) {
|
||||
StopOffset: 2*chunkSize + 8<<10,
|
||||
Value: &ChunkView{FileId: "chunk2", ViewSize: 8 << 10, ViewOffset: 2 * chunkSize, ChunkSize: chunkSize},
|
||||
})
|
||||
views.AppendInterval(&Interval[*ChunkView]{
|
||||
StartOffset: 2*chunkSize + 8<<10,
|
||||
StopOffset: 2*chunkSize + 16<<10,
|
||||
Value: &ChunkView{FileId: "chunk3", ViewSize: 8 << 10, ViewOffset: 2*chunkSize + 8<<10, ChunkSize: chunkSize, CipherKey: []byte("key")},
|
||||
})
|
||||
|
||||
reader := NewChunkReaderAtFromClient(context.Background(), rc, views, 3*chunkSize, 0)
|
||||
buf := make([]byte, chunkSize+16<<10)
|
||||
reader := NewChunkReaderAtFromClient(context.Background(), rc, views, 4*chunkSize, 0)
|
||||
buf := make([]byte, chunkSize+24<<10)
|
||||
if n, err := reader.ReadAt(buf, chunkSize-8<<10); err != nil || n != len(buf) {
|
||||
t.Fatalf("window read: n=%d err=%v", n, err)
|
||||
}
|
||||
// buf holds [56KiB, 136KiB): chunk0's tail, chunk1, chunk2's head.
|
||||
// buf holds [56KiB, 144KiB): chunk0's tail, chunk1, chunk2's and
|
||||
// chunk3's heads.
|
||||
for i, b := range buf {
|
||||
want := byte('1')
|
||||
if i < 8<<10 {
|
||||
want = '0'
|
||||
} else if i >= 16<<10+chunkSize {
|
||||
want = '3'
|
||||
} else if i >= 8<<10+chunkSize {
|
||||
want = '2'
|
||||
}
|
||||
@@ -288,6 +297,9 @@ func TestChunkReadAtClippedViewsFetchOnlyCoveredParts(t *testing.T) {
|
||||
{fileId: "chunk0", isFullChunk: false, offset: chunkSize - 8<<10, size: 8 << 10},
|
||||
{fileId: "chunk1", isFullChunk: true, offset: 0, size: chunkSize},
|
||||
{fileId: "chunk2", isFullChunk: false, offset: 0, size: 8 << 10},
|
||||
// partial view, but a ciphered chunk downloads whole either way and
|
||||
// the shared path decrypts once for every buffer
|
||||
{fileId: "chunk3", isFullChunk: true, offset: 0, size: chunkSize},
|
||||
}
|
||||
got := map[string]recordedFetch{}
|
||||
for _, f := range *fetches {
|
||||
|
||||
@@ -100,9 +100,11 @@ func (rc *ReaderCache) MaybeCache(chunkViews *Interval[*ChunkView], count int) {
|
||||
// abort when slots are filled
|
||||
return
|
||||
}
|
||||
if !chunkView.IsFullChunk() {
|
||||
// the view is clipped to part of the chunk, so the rest is never
|
||||
// read; prefetching it whole would download bytes nobody needs
|
||||
if chunkView.CipherKey == nil && !chunkView.IsFullChunk() {
|
||||
// the view is clipped to part of the chunk and will be
|
||||
// range-fetched, so prefetching it whole would download bytes
|
||||
// nobody needs; a ciphered partial view needs the whole blob
|
||||
// anyway and is worth prefetching
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user