This commit is contained in:
chrislu
2025-11-23 12:43:08 -08:00
parent 48a2ddf6f8
commit 221252d34e
7 changed files with 389 additions and 178 deletions
@@ -168,8 +168,8 @@ public class SeaweedOutputStream extends OutputStream {
totalBytesWritten += length;
if (path.contains("parquet")) {
LOG.info("[DEBUG-2024] ✍️ write({} bytes): totalSoFar={} position={} bufferPos={}, file={}",
length, totalBytesWritten, position, buffer.position(),
LOG.info("[DEBUG-2024] ✍️ write({} bytes): totalSoFar={} position={} bufferPos={}, file={}",
length, totalBytesWritten, position, buffer.position(),
path.substring(path.lastIndexOf('/') + 1));
}
@@ -24,7 +24,7 @@ public class SeaweedRead {
// returns bytesRead
public static long read(FilerClient filerClient, List<VisibleInterval> visibleIntervals,
final long position, final ByteBuffer buf, final long fileSize) throws IOException {
final long position, final ByteBuffer buf, final long fileSize) throws IOException {
List<ChunkView> chunkViews = viewFromVisibles(visibleIntervals, position, buf.remaining());
@@ -51,7 +51,7 @@ public class SeaweedRead {
}
}
//TODO parallel this
// TODO parallel this
long readCount = 0;
long startOffset = position;
for (ChunkView chunkView : chunkViews) {
@@ -59,7 +59,7 @@ public class SeaweedRead {
if (startOffset < chunkView.logicOffset) {
long gap = chunkView.logicOffset - startOffset;
LOG.debug("zero [{},{})", startOffset, startOffset + gap);
buf.position(buf.position()+ (int)gap);
buf.position(buf.position() + (int) gap);
readCount += gap;
startOffset += gap;
}
@@ -86,7 +86,7 @@ public class SeaweedRead {
if (startOffset < limit) {
long gap = limit - startOffset;
LOG.debug("zero2 [{},{})", startOffset, startOffset + gap);
buf.position(buf.position()+ (int)gap);
buf.position(buf.position() + (int) gap);
readCount += gap;
startOffset += gap;
}
@@ -94,7 +94,8 @@ public class SeaweedRead {
return readCount;
}
private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
private static int readChunkView(FilerClient filerClient, long startOffset, ByteBuffer buf, ChunkView chunkView,
FilerProto.Locations locations) throws IOException {
byte[] chunkData = chunkCache.getChunk(chunkView.fileId);
@@ -105,13 +106,15 @@ public class SeaweedRead {
int len = (int) chunkView.size - (int) (startOffset - chunkView.logicOffset);
LOG.debug("readChunkView fid:{} chunkData.length:{} chunkView.offset:{} chunkView[{};{}) startOffset:{}",
chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset, chunkView.logicOffset + chunkView.size, startOffset);
chunkView.fileId, chunkData.length, chunkView.offset, chunkView.logicOffset,
chunkView.logicOffset + chunkView.size, startOffset);
buf.put(chunkData, (int) (startOffset - chunkView.logicOffset + chunkView.offset), len);
return len;
}
public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView, FilerProto.Locations locations) throws IOException {
public static byte[] doFetchFullChunkData(FilerClient filerClient, ChunkView chunkView,
FilerProto.Locations locations) throws IOException {
byte[] data = null;
IOException lastException = null;
@@ -214,8 +217,7 @@ public class SeaweedRead {
chunkStart,
isFullChunk,
chunk.cipherKey,
chunk.isCompressed
));
chunk.isCompressed));
}
}
return views;
@@ -242,7 +244,7 @@ public class SeaweedRead {
long chunksSize = totalSize(entry.getChunksList());
long attrSize = entry.getAttributes().getFileSize();
long finalSize = Math.max(chunksSize, attrSize);
LOG.info("Calculated file size: {} (chunks: {}, attr: {}, #chunks: {})",
LOG.info("Calculated file size: {} (chunks: {}, attr: {}, #chunks: {})",
finalSize, chunksSize, attrSize, entry.getChunksCount());
return finalSize;
}
@@ -268,7 +270,8 @@ public class SeaweedRead {
public final byte[] cipherKey;
public final boolean isCompressed;
public VisibleInterval(long start, long stop, String fileId, long modifiedTime, long chunkOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
public VisibleInterval(long start, long stop, String fileId, long modifiedTime, long chunkOffset,
boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
this.start = start;
this.stop = stop;
this.modifiedTime = modifiedTime;
@@ -302,7 +305,8 @@ public class SeaweedRead {
public final byte[] cipherKey;
public final boolean isCompressed;
public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey, boolean isCompressed) {
public ChunkView(String fileId, long offset, long size, long logicOffset, boolean isFullChunk, byte[] cipherKey,
boolean isCompressed) {
this.fileId = fileId;
this.offset = offset;
this.size = size;