Github-Mirror/seaweedfs
1
0
Fork 0
mirror of https://github.com/seaweedfs/seaweedfs.git synced 2026-09-19 13:00:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: reduce write() logging verbosity, add summary stats

Browse Source 
Previous run showed Parquet writes byte-by-byte (hundreds of 1-byte writes),
flooding logs and getting truncated. This prevented seeing the full picture.

Changes:
1. Only log writes >= 20 bytes (skip byte-by-byte metadata writes)
2. Track writeCallCount to see total number of write() invocations
3. Show writeCallCount in close() summary logs

This will show:
- Large data writes clearly (26, 34, 41, 67 bytes, etc.)
- Total bytes written vs total calls (e.g., 684 bytes in 200+ calls)
- Whether ALL bytes Parquet wrote actually reached close()

If totalBytesWritten=684 at close, Parquet only sent 684 bytes.
If totalBytesWritten=762 at close, Parquet sent all 762 bytes but we lost 78.

Next run will definitively answer: Does Parquet write 684 or 762 bytes total?
This commit is contained in:
chrislu
2025-11-23 12:46:36 -08:00
parent 221252d34e
commit 885354bb19
1 changed files with 10 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
other/java/client/src/main/java/seaweedfs/client/SeaweedOutputStream.java
+10 -8
View File
@@ -35,6 +35,7 @@ public class SeaweedOutputStream extends OutputStream {
private String replication = "";
private String collection = "";
private long totalBytesWritten = 0; // Track total bytes for debugging
private long writeCallCount = 0; // Track number of write() calls
public SeaweedOutputStream(FilerClient filerClient, final String fullpath) {
this(filerClient, fullpath, "");
@@ -149,7 +150,6 @@ public class SeaweedOutputStream extends OutputStream {
@Override
public void write(final int byteVal) throws IOException {
LOG.debug("[DEBUG-2024] ✍️ write(int): 1 byte, path={}", path);
write(new byte[] { (byte) (byteVal & 0xFF) });
}
@@ -167,9 +167,11 @@ public class SeaweedOutputStream extends OutputStream {
}
totalBytesWritten += length;
if (path.contains("parquet")) {
LOG.info("[DEBUG-2024] ✍️ write({} bytes): totalSoFar={} position={} bufferPos={}, file={}",
length, totalBytesWritten, position, buffer.position(),
writeCallCount++;
// Only log significant writes to avoid flooding logs with byte-by-byte writes
if (path.contains("parquet") && length >= 20) {
LOG.info("[DEBUG-2024] ✍️ write({} bytes): totalSoFar={} writeCalls={} position={} bufferPos={}, file={}",
length, totalBytesWritten, writeCallCount, position, buffer.position(),
path.substring(path.lastIndexOf('/') + 1));
}
@@ -226,13 +228,13 @@ public class SeaweedOutputStream extends OutputStream {
}
int bufferPosBeforeFlush = buffer.position();
LOG.info("[DEBUG-2024] 🔒 close START: path={} position={} buffer.position()={} totalBytesWritten={}",
path, position, bufferPosBeforeFlush, totalBytesWritten);
LOG.info("[DEBUG-2024] 🔒 close START: path={} position={} buffer.position()={} totalBytesWritten={} writeCalls={}",
path, position, bufferPosBeforeFlush, totalBytesWritten, writeCallCount);
try {
flushInternal();
threadExecutor.shutdown();
LOG.info("[DEBUG-2024] ✅ close END: path={} finalPosition={} totalBytesWritten={} (buffer had {} bytes)",
path, position, totalBytesWritten, bufferPosBeforeFlush);
LOG.info("[DEBUG-2024] ✅ close END: path={} finalPosition={} totalBytesWritten={} writeCalls={} (buffer had {} bytes)",
path, position, totalBytesWritten, writeCallCount, bufferPosBeforeFlush);
} finally {
lastError = new IOException("Stream is closed!");
ByteBufferPool.release(buffer);