mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-09-09 16:10:44 +02:00
chrislu
a3cf4eb843
Added comprehensive logging to identify why Parquet files fail with 'EOFException: Still have: 78 bytes left'. Key additions: 1. SeaweedHadoopOutputStream constructor logging with 🔧 marker - Shows when output streams are created - Logs path, position, bufferSize, replication 2. totalBytesWritten counter in SeaweedOutputStream - Tracks cumulative bytes written via write() calls - Helps identify if Parquet wrote 762 bytes but only 684 reached chunks 3. Enhanced close() logging with 🔒 and ✅ markers - Shows totalBytesWritten vs position vs buffer.position() - If totalBytesWritten=762 but position=684, write submission failed - If buffer.position()=78 at close, buffer wasn't flushed Expected scenarios in next run: A) Stream never created → No 🔧 log for .parquet files B) Write failed → totalBytesWritten=762 but position=684 C) Buffer not flushed → buffer.position()=78 at close D) All correct → totalBytesWritten=position=684, but Parquet expects 762 This will pinpoint whether the issue is in: - Stream creation/lifecycle - Write submission - Buffer flushing - Or Parquet's internal state
47 lines
924 B
Bash
Executable File
47 lines
924 B
Bash
Executable File
#!/bin/bash
|
|
|
|
set -e
|
|
|
|
echo "=== SeaweedFS Spark Integration Tests Runner ==="
|
|
echo ""
|
|
|
|
# Check if SeaweedFS is running
|
|
check_seaweedfs() {
|
|
if curl -f http://localhost:8888/ > /dev/null 2>&1; then
|
|
echo "✓ SeaweedFS filer is accessible at http://localhost:8888"
|
|
return 0
|
|
else
|
|
echo "✗ SeaweedFS filer is not accessible"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Main
|
|
if ! check_seaweedfs; then
|
|
echo ""
|
|
echo "Please start SeaweedFS first. You can use:"
|
|
echo " cd test/java/spark && docker-compose up -d"
|
|
echo "Or:"
|
|
echo " make docker-up"
|
|
exit 1
|
|
fi
|
|
|
|
echo ""
|
|
echo "Running Spark integration tests..."
|
|
echo ""
|
|
|
|
export SEAWEEDFS_TEST_ENABLED=true
|
|
export SEAWEEDFS_FILER_HOST=localhost
|
|
export SEAWEEDFS_FILER_PORT=8888
|
|
export SEAWEEDFS_FILER_GRPC_PORT=18888
|
|
|
|
# Run tests
|
|
mvn test "$@"
|
|
|
|
echo ""
|
|
echo "✓ Test run completed"
|
|
echo "View detailed reports in: target/surefire-reports/"
|
|
|
|
|
|
|