diff --git a/.gitignore b/.gitignore index 2306882fe..9a3bf80cd 100644 --- a/.gitignore +++ b/.gitignore @@ -116,7 +116,6 @@ test/s3/versioning/weed-test.log /docker/admin_integration/data docker/agent_pub_record docker/admin_integration/weed-local -/seaweedfs-rdma-sidecar/bin /test/s3/encryption/filerldb2 /test/s3/sse/filerldb2 test/s3/sse/weed-test.log diff --git a/seaweedfs-rdma-sidecar/.dockerignore b/seaweedfs-rdma-sidecar/.dockerignore deleted file mode 100644 index 3989eb5bd..000000000 --- a/seaweedfs-rdma-sidecar/.dockerignore +++ /dev/null @@ -1,65 +0,0 @@ -# Git -.git -.gitignore -.gitmodules - -# Documentation -*.md -docs/ - -# Development files -.vscode/ -.idea/ -*.swp -*.swo -*~ - -# OS generated files -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -ehthumbs.db -Thumbs.db - -# Build artifacts -# bin/ (commented out for Docker build - needed for mount container) -# target/ (commented out for Docker build) -*.exe -*.dll -*.so -*.dylib - -# Go specific -vendor/ -*.test -*.prof -go.work -go.work.sum - -# Rust specific -Cargo.lock -# rdma-engine/target/ (commented out for Docker build) -*.pdb - -# Docker -Dockerfile* -docker-compose*.yml -.dockerignore - -# Test files (tests/ needed for integration test container) -# tests/ -# scripts/ (commented out for Docker build - needed for mount container) -*.log - -# Temporary files -tmp/ -temp/ -*.tmp -*.temp - -# IDE and editor files -*.sublime-* -.vscode/ -.idea/ diff --git a/seaweedfs-rdma-sidecar/CORRECT-SIDECAR-APPROACH.md b/seaweedfs-rdma-sidecar/CORRECT-SIDECAR-APPROACH.md deleted file mode 100644 index 743128ba8..000000000 --- a/seaweedfs-rdma-sidecar/CORRECT-SIDECAR-APPROACH.md +++ /dev/null @@ -1,196 +0,0 @@ -# โœ… Correct RDMA Sidecar Approach - Simple Parameter-Based - -## ๐ŸŽฏ **You're Right - Simplified Architecture** - -The RDMA sidecar should be **simple** and just take the volume server address as a parameter. The volume lookup complexity should stay in `weed mount`, not in the sidecar. - -## ๐Ÿ—๏ธ **Correct Architecture** - -### **1. weed mount (Client Side) - Does Volume Lookup** -```go -// File: weed/mount/filehandle_read.go (integration point) -func (fh *FileHandle) tryRDMARead(ctx context.Context, buff []byte, offset int64) (int64, int64, error) { - entry := fh.GetEntry() - - for _, chunk := range entry.GetEntry().Chunks { - if offset >= chunk.Offset && offset < chunk.Offset+int64(chunk.Size) { - // Parse chunk info - volumeID, needleID, cookie, err := ParseFileId(chunk.FileId) - if err != nil { - return 0, 0, err - } - - // ๐Ÿ” VOLUME LOOKUP (in weed mount, not sidecar) - volumeServerAddr, err := fh.wfs.lookupVolumeServer(ctx, volumeID) - if err != nil { - return 0, 0, err - } - - // ๐Ÿš€ SIMPLE RDMA REQUEST WITH VOLUME SERVER PARAMETER - data, isRDMA, err := fh.wfs.rdmaClient.ReadNeedleFromServer( - ctx, volumeServerAddr, volumeID, needleID, cookie, chunkOffset, readSize) - - return int64(copy(buff, data)), time.Now().UnixNano(), nil - } - } -} -``` - -### **2. RDMA Mount Client - Passes Volume Server Address** -```go -// File: weed/mount/rdma_client.go (modify existing) -func (c *RDMAMountClient) ReadNeedleFromServer(ctx context.Context, volumeServerAddr string, volumeID uint32, needleID uint64, cookie uint32, offset, size uint64) ([]byte, bool, error) { - // Simple HTTP request with volume server as parameter - reqURL := fmt.Sprintf("http://%s/rdma/read", c.sidecarAddr) - - requestBody := map[string]interface{}{ - "volume_server": volumeServerAddr, // โ† KEY: Pass volume server address - "volume_id": volumeID, - "needle_id": needleID, - "cookie": cookie, - "offset": offset, - "size": size, - } - - // POST request with volume server parameter - jsonBody, err := json.Marshal(requestBody) - if err != nil { - return nil, false, fmt.Errorf("failed to marshal request body: %w", err) - } - resp, err := c.httpClient.Post(reqURL, "application/json", bytes.NewBuffer(jsonBody)) - if err != nil { - return nil, false, fmt.Errorf("http post to sidecar: %w", err) - } -} -``` - -### **3. RDMA Sidecar - Simple, No Lookup Logic** -```go -// File: seaweedfs-rdma-sidecar/cmd/demo-server/main.go -func (s *DemoServer) rdmaReadHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse request body - var req struct { - VolumeServer string `json:"volume_server"` // โ† Receive volume server address - VolumeID uint32 `json:"volume_id"` - NeedleID uint64 `json:"needle_id"` - Cookie uint32 `json:"cookie"` - Offset uint64 `json:"offset"` - Size uint64 `json:"size"` - } - - if err := json.NewDecoder(r.Body).Decode(&req); err != nil { - http.Error(w, "Invalid request", http.StatusBadRequest) - return - } - - s.logger.WithFields(logrus.Fields{ - "volume_server": req.VolumeServer, // โ† Use provided volume server - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - }).Info("๐Ÿ“– Processing RDMA read with volume server parameter") - - // ๐Ÿš€ SIMPLE: Use the provided volume server address - // No complex lookup logic needed! - resp, err := s.rdmaClient.ReadFromVolumeServer(r.Context(), req.VolumeServer, req.VolumeID, req.NeedleID, req.Cookie, req.Offset, req.Size) - - if err != nil { - http.Error(w, fmt.Sprintf("RDMA read failed: %v", err), http.StatusInternalServerError) - return - } - - // Return binary data - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("X-RDMA-Used", "true") - w.Write(resp.Data) -} -``` - -### **4. Volume Lookup in weed mount (Where it belongs)** -```go -// File: weed/mount/weedfs.go (add method) -func (wfs *WFS) lookupVolumeServer(ctx context.Context, volumeID uint32) (string, error) { - // Use existing SeaweedFS volume lookup logic - vid := fmt.Sprintf("%d", volumeID) - - // Query master server for volume location - locations, err := operation.LookupVolumeId(wfs.getMasterFn(), wfs.option.GrpcDialOption, vid) - if err != nil { - return "", fmt.Errorf("volume lookup failed: %w", err) - } - - if len(locations.Locations) == 0 { - return "", fmt.Errorf("no locations found for volume %d", volumeID) - } - - // Return first available location (or implement smart selection) - return locations.Locations[0].Url, nil -} -``` - -## ๐ŸŽฏ **Key Differences from Over-Complicated Approach** - -### **โŒ Over-Complicated (What I Built Before):** -- โŒ Sidecar does volume lookup -- โŒ Sidecar has master client integration -- โŒ Sidecar has volume location caching -- โŒ Sidecar forwards requests to remote sidecars -- โŒ Complex distributed logic in sidecar - -### **โœ… Correct Simple Approach:** -- โœ… **weed mount** does volume lookup (where it belongs) -- โœ… **weed mount** passes volume server address to sidecar -- โœ… **Sidecar** is simple and stateless -- โœ… **Sidecar** just does local RDMA read for given server -- โœ… **No complex distributed logic in sidecar** - -## ๐Ÿš€ **Request Flow (Corrected)** - -1. **User Application** โ†’ `read()` system call -2. **FUSE** โ†’ `weed mount` WFS.Read() -3. **weed mount** โ†’ Volume lookup: "Where is volume 7?" -4. **SeaweedFS Master** โ†’ "Volume 7 is on server-B:8080" -5. **weed mount** โ†’ HTTP POST to sidecar: `{volume_server: "server-B:8080", volume: 7, needle: 12345}` -6. **RDMA Sidecar** โ†’ Connect to server-B:8080, do local RDMA read -7. **RDMA Engine** โ†’ Direct memory access to volume file -8. **Response** โ†’ Binary data back to weed mount โ†’ user - -## ๐Ÿ“ **Implementation Changes Needed** - -### **1. Simplify Sidecar (Remove Complex Logic)** -- Remove `DistributedRDMAClient` -- Remove volume lookup logic -- Remove master client integration -- Keep simple RDMA engine communication - -### **2. Add Volume Lookup to weed mount** -- Add `lookupVolumeServer()` method to WFS -- Modify `RDMAMountClient` to accept volume server parameter -- Integrate with existing SeaweedFS volume lookup - -### **3. Simple Sidecar API** -``` -POST /rdma/read -{ - "volume_server": "server-B:8080", - "volume_id": 7, - "needle_id": 12345, - "cookie": 0, - "offset": 0, - "size": 4096 -} -``` - -## โœ… **Benefits of Simple Approach** - -- **๐ŸŽฏ Single Responsibility**: Sidecar only does RDMA, weed mount does lookup -- **๐Ÿ”ง Maintainable**: Less complex logic in sidecar -- **โšก Performance**: No extra network hops for volume lookup -- **๐Ÿ—๏ธ Clean Architecture**: Separation of concerns -- **๐Ÿ› Easier Debugging**: Clear responsibility boundaries - -You're absolutely right - this is much cleaner! The sidecar should be a simple RDMA accelerator, not a distributed system coordinator. diff --git a/seaweedfs-rdma-sidecar/CURRENT-STATUS.md b/seaweedfs-rdma-sidecar/CURRENT-STATUS.md deleted file mode 100644 index e8f53dc1d..000000000 --- a/seaweedfs-rdma-sidecar/CURRENT-STATUS.md +++ /dev/null @@ -1,165 +0,0 @@ -# SeaweedFS RDMA Sidecar - Current Status Summary - -## ๐ŸŽ‰ **IMPLEMENTATION COMPLETE** -**Status**: โœ… **READY FOR PRODUCTION** (Mock Mode) / ๐Ÿ”„ **READY FOR HARDWARE INTEGRATION** - ---- - -## ๐Ÿ“Š **What's Working Right Now** - -### โœ… **Complete Integration Pipeline** -- **SeaweedFS Mount** โ†’ **Go Sidecar** โ†’ **Rust Engine** โ†’ **Mock RDMA** -- End-to-end data flow with proper error handling -- Zero-copy page cache optimization -- Connection pooling for performance - -### โœ… **Production-Ready Components** -- HTTP API with RESTful endpoints -- Robust health checks and monitoring -- Docker multi-service orchestration -- Comprehensive error handling and fallback -- Volume lookup and server discovery - -### โœ… **Performance Features** -- **Zero-Copy**: Direct kernel page cache population -- **Connection Pooling**: Reused IPC connections -- **Async Operations**: Non-blocking I/O throughout -- **Metrics**: Detailed performance monitoring - -### โœ… **Code Quality** -- All GitHub PR review comments addressed -- Memory-safe operations (no dangerous channel closes) -- Proper file ID parsing using SeaweedFS functions -- RESTful API design with correct HTTP methods - ---- - -## ๐Ÿ”„ **What's Mock/Simulated** - -### ๐ŸŸก **Mock RDMA Engine** (Rust) -- **Location**: `rdma-engine/src/rdma.rs` -- **Function**: Simulates RDMA hardware operations -- **Data**: Generates pattern data (0,1,2...255,0,1,2...) -- **Performance**: Realistic latency simulation (150ns reads) - -### ๐ŸŸก **Simulated Hardware** -- **Device Info**: Mock Mellanox ConnectX-5 capabilities -- **Memory Regions**: Fake registration without HCA -- **Transfers**: Pattern generation instead of network transfer -- **Completions**: Synthetic work completions - ---- - -## ๐Ÿ“ˆ **Current Performance** -- **Throughput**: ~403 operations/second -- **Latency**: ~2.48ms average (mock overhead) -- **Success Rate**: 100% in integration tests -- **Memory Usage**: Optimized with zero-copy - ---- - -## ๐Ÿ—๏ธ **Architecture Overview** - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SeaweedFS โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Go Sidecar โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Rust Engine โ”‚ -โ”‚ Mount Client โ”‚ โ”‚ HTTP Server โ”‚ โ”‚ Mock RDMA โ”‚ -โ”‚ (REAL) โ”‚ โ”‚ (REAL) โ”‚ โ”‚ (MOCK) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ - File ID Parse โ”‚ โ”‚ - Zero-Copy โ”‚ โ”‚ - UCX Ready โ”‚ -โ”‚ - Volume Lookup โ”‚ โ”‚ - Conn Pooling โ”‚ โ”‚ - Memory Mgmt โ”‚ -โ”‚ - HTTP Fallback โ”‚ โ”‚ - Health Checks โ”‚ โ”‚ - IPC Protocol โ”‚ -โ”‚ - Error Handlingโ”‚ โ”‚ - REST API โ”‚ โ”‚ - Async Ops โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - ---- - -## ๐Ÿ”ง **Key Files & Locations** - -### **Core Integration** -- `weed/mount/filehandle_read.go` - RDMA read integration in FUSE -- `weed/mount/rdma_client.go` - Mount client RDMA communication -- `cmd/demo-server/main.go` - Main RDMA sidecar HTTP server - -### **RDMA Engine** -- `rdma-engine/src/rdma.rs` - Mock RDMA implementation -- `rdma-engine/src/ipc.rs` - IPC protocol with Go sidecar -- `pkg/rdma/client.go` - Go client for RDMA engine - -### **Configuration** -- `docker-compose.mount-rdma.yml` - Complete integration test setup -- `go.mod` - Dependencies with local SeaweedFS replacement - ---- - -## ๐Ÿš€ **Ready For Next Steps** - -### **Immediate Capability** -- โœ… **Development**: Full testing without RDMA hardware -- โœ… **Integration Testing**: Complete pipeline validation -- โœ… **Performance Benchmarking**: Baseline metrics -- โœ… **CI/CD**: Mock mode for automated testing - -### **Production Transition** -- ๐Ÿ”„ **Hardware Integration**: Replace mock with UCX library -- ๐Ÿ”„ **Real Data Transfer**: Remove pattern generation -- ๐Ÿ”„ **Device Detection**: Enumerate actual RDMA NICs -- ๐Ÿ”„ **Performance Optimization**: Hardware-specific tuning - ---- - -## ๐Ÿ“‹ **Commands to Resume Work** - -### **Start Development Environment** -```bash -# Navigate to your seaweedfs-rdma-sidecar directory -cd /path/to/your/seaweedfs/seaweedfs-rdma-sidecar - -# Build components -go build -o bin/demo-server ./cmd/demo-server -cargo build --manifest-path rdma-engine/Cargo.toml - -# Run integration tests -docker-compose -f docker-compose.mount-rdma.yml up -``` - -### **Test Current Implementation** -```bash -# Test sidecar HTTP API -curl http://localhost:8081/health -curl http://localhost:8081/stats - -# Test RDMA read -curl "http://localhost:8081/read?volume=1&needle=123&cookie=456&offset=0&size=1024&volume_server=http://localhost:8080" -``` - ---- - -## ๐ŸŽฏ **Success Metrics Achieved** - -- โœ… **Functional**: Complete RDMA integration pipeline -- โœ… **Reliable**: Robust error handling and fallback -- โœ… **Performant**: Zero-copy and connection pooling -- โœ… **Testable**: Comprehensive mock implementation -- โœ… **Maintainable**: Clean code with proper documentation -- โœ… **Scalable**: Async operations and pooling -- โœ… **Production-Ready**: All review comments addressed - ---- - -## ๐Ÿ“š **Documentation** - -- `FUTURE-WORK-TODO.md` - Next steps for hardware integration -- `DOCKER-TESTING.md` - Integration testing guide -- `docker-compose.mount-rdma.yml` - Complete test environment -- GitHub PR reviews - All issues addressed and documented - ---- - -**๐Ÿ† ACHIEVEMENT**: Complete RDMA sidecar architecture with production-ready infrastructure and seamless mock-to-real transition path! - -**Next**: Follow `FUTURE-WORK-TODO.md` to replace mock with real UCX hardware integration. diff --git a/seaweedfs-rdma-sidecar/DOCKER-TESTING.md b/seaweedfs-rdma-sidecar/DOCKER-TESTING.md deleted file mode 100644 index 88ea1971d..000000000 --- a/seaweedfs-rdma-sidecar/DOCKER-TESTING.md +++ /dev/null @@ -1,290 +0,0 @@ -# ๐Ÿณ Docker Integration Testing Guide - -This guide provides comprehensive Docker-based integration testing for the SeaweedFS RDMA sidecar system. - -## ๐Ÿ—๏ธ Architecture - -The Docker Compose setup includes: - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SeaweedFS Master โ”‚ โ”‚ SeaweedFS Volume โ”‚ โ”‚ Rust RDMA โ”‚ -โ”‚ :9333 โ”‚โ—„โ”€โ”€โ–บโ”‚ :8080 โ”‚ โ”‚ Engine โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ - โ–ผ โ–ผ -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Go RDMA Sidecar โ”‚โ—„โ”€โ”€โ–บโ”‚ Unix Socket โ”‚โ—„โ”€โ”€โ–บโ”‚ Integration โ”‚ -โ”‚ :8081 โ”‚ โ”‚ /tmp/rdma.sock โ”‚ โ”‚ Test Suite โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -## ๐Ÿš€ Quick Start - -### 1. Start All Services - -```bash -# Using the helper script (recommended) -./tests/docker-test-helper.sh start - -# Or using docker-compose directly -docker-compose up -d -``` - -### 2. Run Integration Tests - -```bash -# Run the complete test suite -./tests/docker-test-helper.sh test - -# Or run tests manually -docker-compose run --rm integration-tests -``` - -### 3. Interactive Testing - -```bash -# Open a shell in the test container -./tests/docker-test-helper.sh shell - -# Inside the container, you can run: -./test-rdma ping -./test-rdma capabilities -./test-rdma read --volume 1 --needle 12345 --size 1024 -curl http://rdma-sidecar:8081/health -curl http://rdma-sidecar:8081/stats -``` - -## ๐Ÿ“‹ Test Helper Commands - -The `docker-test-helper.sh` script provides convenient commands: - -```bash -# Service Management -./tests/docker-test-helper.sh start # Start all services -./tests/docker-test-helper.sh stop # Stop all services -./tests/docker-test-helper.sh clean # Stop and clean volumes - -# Testing -./tests/docker-test-helper.sh test # Run integration tests -./tests/docker-test-helper.sh shell # Interactive testing shell - -# Monitoring -./tests/docker-test-helper.sh status # Check service health -./tests/docker-test-helper.sh logs # Show all logs -./tests/docker-test-helper.sh logs rdma-engine # Show specific service logs -``` - -## ๐Ÿงช Test Coverage - -The integration test suite covers: - -### โœ… Core Components -- **SeaweedFS Master**: Cluster leadership and status -- **SeaweedFS Volume Server**: Volume operations and health -- **Rust RDMA Engine**: Socket communication and operations -- **Go RDMA Sidecar**: HTTP API and RDMA integration - -### โœ… Integration Points -- **IPC Communication**: Unix socket + MessagePack protocol -- **RDMA Operations**: Ping, capabilities, read operations -- **HTTP API**: All sidecar endpoints and error handling -- **Fallback Logic**: RDMA โ†’ HTTP fallback behavior - -### โœ… Performance Testing -- **Direct RDMA Benchmarks**: Engine-level performance -- **Sidecar Benchmarks**: End-to-end performance -- **Latency Measurements**: Operation timing validation -- **Throughput Testing**: Operations per second - -## ๐Ÿ”ง Service Details - -### SeaweedFS Master -- **Port**: 9333 -- **Health Check**: `/cluster/status` -- **Data**: Persistent volume `master-data` - -### SeaweedFS Volume Server -- **Port**: 8080 -- **Health Check**: `/status` -- **Data**: Persistent volume `volume-data` -- **Depends on**: SeaweedFS Master - -### Rust RDMA Engine -- **Socket**: `/tmp/rdma-engine.sock` -- **Mode**: Mock RDMA (development) -- **Health Check**: Socket existence -- **Privileged**: Yes (for RDMA access) - -### Go RDMA Sidecar -- **Port**: 8081 -- **Health Check**: `/health` -- **API Endpoints**: `/stats`, `/read`, `/benchmark` -- **Depends on**: RDMA Engine, Volume Server - -### Test Client -- **Purpose**: Integration testing and interactive debugging -- **Tools**: curl, jq, test-rdma binary -- **Environment**: All service URLs configured - -## ๐Ÿ“Š Expected Test Results - -### โœ… Successful Output Example - -``` -=============================================== -๐Ÿš€ SEAWEEDFS RDMA INTEGRATION TEST SUITE -=============================================== - -๐Ÿ”ต Waiting for SeaweedFS Master to be ready... -โœ… SeaweedFS Master is ready -โœ… SeaweedFS Master is leader and ready - -๐Ÿ”ต Waiting for SeaweedFS Volume Server to be ready... -โœ… SeaweedFS Volume Server is ready -Volume Server Version: 3.60 - -๐Ÿ”ต Checking RDMA engine socket... -โœ… RDMA engine socket exists -๐Ÿ”ต Testing RDMA engine ping... -โœ… RDMA engine ping successful - -๐Ÿ”ต Waiting for RDMA Sidecar to be ready... -โœ… RDMA Sidecar is ready -โœ… RDMA Sidecar is healthy -RDMA Status: true - -๐Ÿ”ต Testing needle read via sidecar... -โœ… Sidecar needle read successful -โš ๏ธ HTTP fallback used. Duration: 2.48ms - -๐Ÿ”ต Running sidecar performance benchmark... -โœ… Sidecar benchmark completed -Benchmark Results: - RDMA Operations: 5 - HTTP Operations: 0 - Average Latency: 2.479ms - Operations/sec: 403.2 - -=============================================== -๐ŸŽ‰ ALL INTEGRATION TESTS COMPLETED! -=============================================== -``` - -## ๐Ÿ› Troubleshooting - -### Service Not Starting - -```bash -# Check service logs -./tests/docker-test-helper.sh logs [service-name] - -# Check container status -docker-compose ps - -# Restart specific service -docker-compose restart [service-name] -``` - -### RDMA Engine Issues - -```bash -# Check socket permissions -docker-compose exec rdma-engine ls -la /tmp/rdma/rdma-engine.sock - -# Check RDMA engine logs -./tests/docker-test-helper.sh logs rdma-engine - -# Test socket directly -docker-compose exec test-client ./test-rdma ping -``` - -### Sidecar Connection Issues - -```bash -# Test sidecar health directly -curl http://localhost:8081/health - -# Check sidecar logs -./tests/docker-test-helper.sh logs rdma-sidecar - -# Verify environment variables -docker-compose exec rdma-sidecar env | grep RDMA -``` - -### Volume Server Issues - -```bash -# Check SeaweedFS status -curl http://localhost:9333/cluster/status -curl http://localhost:8080/status - -# Check volume server logs -./tests/docker-test-helper.sh logs seaweedfs-volume -``` - -## ๐Ÿ” Manual Testing Examples - -### Test RDMA Engine Directly - -```bash -# Enter test container -./tests/docker-test-helper.sh shell - -# Test RDMA operations -./test-rdma ping --socket /tmp/rdma-engine.sock -./test-rdma capabilities --socket /tmp/rdma-engine.sock -./test-rdma read --socket /tmp/rdma-engine.sock --volume 1 --needle 12345 -./test-rdma bench --socket /tmp/rdma-engine.sock --iterations 10 -``` - -### Test Sidecar HTTP API - -```bash -# Health and status -curl http://rdma-sidecar:8081/health | jq '.' -curl http://rdma-sidecar:8081/stats | jq '.' - -# Needle operations -curl "http://rdma-sidecar:8081/read?volume=1&needle=12345&size=1024" | jq '.' - -# Benchmarking -curl "http://rdma-sidecar:8081/benchmark?iterations=5&size=2048" | jq '.benchmark_results' -``` - -### Test SeaweedFS Integration - -```bash -# Check cluster status -curl http://seaweedfs-master:9333/cluster/status | jq '.' - -# Check volume status -curl http://seaweedfs-volume:8080/status | jq '.' - -# List volumes -curl http://seaweedfs-master:9333/vol/status | jq '.' -``` - -## ๐Ÿš€ Production Deployment - -This Docker setup can be adapted for production by: - -1. **Replacing Mock RDMA**: Switch to `real-ucx` feature in Rust -2. **RDMA Hardware**: Add RDMA device mappings and capabilities -3. **Security**: Remove privileged mode, add proper user/group mapping -4. **Scaling**: Use Docker Swarm or Kubernetes for orchestration -5. **Monitoring**: Add Prometheus metrics and Grafana dashboards -6. **Persistence**: Configure proper volume management - -## ๐Ÿ“š Additional Resources - -- [Main README](README.md) - Complete project overview -- [Docker Compose Reference](https://docs.docker.com/compose/) -- [SeaweedFS Documentation](https://github.com/seaweedfs/seaweedfs/wiki) -- [UCX Documentation](https://github.com/openucx/ucx) - ---- - -**๐Ÿณ Happy Docker Testing!** - -For issues or questions, please check the logs first and refer to the troubleshooting section above. diff --git a/seaweedfs-rdma-sidecar/Dockerfile.integration-test b/seaweedfs-rdma-sidecar/Dockerfile.integration-test deleted file mode 100644 index 8e9d6610e..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.integration-test +++ /dev/null @@ -1,25 +0,0 @@ -# Dockerfile for RDMA Mount Integration Tests -FROM ubuntu:22.04 - -# Install dependencies -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - ca-certificates \ - jq \ - bc \ - time \ - util-linux \ - coreutils \ - && rm -rf /var/lib/apt/lists/* - -# Create test directories -RUN mkdir -p /usr/local/bin /test-results - -# Copy test scripts -COPY scripts/run-integration-tests.sh /usr/local/bin/run-integration-tests.sh -COPY scripts/test-rdma-mount.sh /usr/local/bin/test-rdma-mount.sh -RUN chmod +x /usr/local/bin/*.sh - -# Default command -CMD ["/usr/local/bin/run-integration-tests.sh"] diff --git a/seaweedfs-rdma-sidecar/Dockerfile.mount-rdma b/seaweedfs-rdma-sidecar/Dockerfile.mount-rdma deleted file mode 100644 index 425defcc7..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.mount-rdma +++ /dev/null @@ -1,40 +0,0 @@ -# Dockerfile for SeaweedFS Mount with RDMA support -FROM ubuntu:22.04 - -# Install dependencies -RUN apt-get update && apt-get install -y \ - fuse3 \ - curl \ - wget \ - ca-certificates \ - procps \ - util-linux \ - jq \ - && rm -rf /var/lib/apt/lists/* - -# Create necessary directories -RUN mkdir -p /usr/local/bin /mnt/seaweedfs /var/log/seaweedfs - -# Copy SeaweedFS binary (will be built from context) -COPY bin/weed /usr/local/bin/weed -RUN chmod +x /usr/local/bin/weed - -# Copy mount helper scripts -COPY scripts/mount-helper.sh /usr/local/bin/mount-helper.sh -RUN chmod +x /usr/local/bin/mount-helper.sh - -# Create mount point -RUN mkdir -p /mnt/seaweedfs - -# Set up FUSE permissions -RUN echo 'user_allow_other' >> /etc/fuse.conf - -# Health check script -COPY scripts/mount-health-check.sh /usr/local/bin/mount-health-check.sh -RUN chmod +x /usr/local/bin/mount-health-check.sh - -# Expose mount point as volume -VOLUME ["/mnt/seaweedfs"] - -# Default command -CMD ["/usr/local/bin/mount-helper.sh"] diff --git a/seaweedfs-rdma-sidecar/Dockerfile.performance-test b/seaweedfs-rdma-sidecar/Dockerfile.performance-test deleted file mode 100644 index 7ffa81c4f..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.performance-test +++ /dev/null @@ -1,26 +0,0 @@ -# Dockerfile for RDMA Mount Performance Tests -FROM ubuntu:22.04 - -# Install dependencies -RUN apt-get update && apt-get install -y \ - curl \ - wget \ - ca-certificates \ - jq \ - bc \ - time \ - util-linux \ - coreutils \ - fio \ - iozone3 \ - && rm -rf /var/lib/apt/lists/* - -# Create test directories -RUN mkdir -p /usr/local/bin /performance-results - -# Copy test scripts -COPY scripts/run-performance-tests.sh /usr/local/bin/run-performance-tests.sh -RUN chmod +x /usr/local/bin/*.sh - -# Default command -CMD ["/usr/local/bin/run-performance-tests.sh"] diff --git a/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine b/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine deleted file mode 100644 index 539a71bd1..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine +++ /dev/null @@ -1,63 +0,0 @@ -# Multi-stage build for Rust RDMA Engine -FROM rust:1.80-slim AS builder - -# Install build dependencies -RUN apt-get update && apt-get install -y \ - pkg-config \ - libssl-dev \ - libudev-dev \ - build-essential \ - libc6-dev \ - linux-libc-dev \ - && rm -rf /var/lib/apt/lists/* - -# Set work directory -WORKDIR /app - -# Copy Rust project files -COPY rdma-engine/Cargo.toml ./ -COPY rdma-engine/Cargo.lock ./ -COPY rdma-engine/src ./src - -# Build the release binary -RUN cargo build --release - -# Runtime stage -FROM debian:bookworm-slim - -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - && rm -rf /var/lib/apt/lists/* - -# Create app user -RUN useradd -m -u 1001 appuser - -# Set work directory -WORKDIR /app - -# Copy binary from builder stage -COPY --from=builder /app/target/release/rdma-engine-server . - -# Change ownership -RUN chown -R appuser:appuser /app - -# Set default socket path (can be overridden) -ENV RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock - -# Create socket directory with proper permissions (before switching user) -RUN mkdir -p /tmp/rdma && chown -R appuser:appuser /tmp/rdma - -USER appuser - -# Expose any needed ports (none for this service as it uses Unix sockets) -# EXPOSE 18515 - -# Health check - verify both process and socket using environment variable -HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep rdma-engine-server >/dev/null && test -S "$RDMA_SOCKET_PATH" - -# Default command using environment variable -CMD sh -c "./rdma-engine-server --debug --ipc-socket \"$RDMA_SOCKET_PATH\"" diff --git a/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine.simple b/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine.simple deleted file mode 100644 index cbe3edf16..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.rdma-engine.simple +++ /dev/null @@ -1,36 +0,0 @@ -# Simplified Dockerfile for Rust RDMA Engine (using pre-built binary) -FROM debian:bookworm-slim - -# Install runtime dependencies -RUN apt-get update && apt-get install -y \ - ca-certificates \ - libssl3 \ - curl \ - procps \ - && rm -rf /var/lib/apt/lists/* - -# Create app user -RUN useradd -m -u 1001 appuser - -# Set work directory -WORKDIR /app - -# Copy pre-built binary from local build -COPY ./rdma-engine/target/release/rdma-engine-server . - -# Change ownership -RUN chown -R appuser:appuser /app -USER appuser - -# Set default socket path (can be overridden) -ENV RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - -# Create socket directory -RUN mkdir -p /tmp - -# Health check - verify both process and socket using environment variable -HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ - CMD pgrep rdma-engine-server >/dev/null && test -S "$RDMA_SOCKET_PATH" - -# Default command using environment variable -CMD sh -c "./rdma-engine-server --debug --ipc-socket \"$RDMA_SOCKET_PATH\"" diff --git a/seaweedfs-rdma-sidecar/Dockerfile.sidecar b/seaweedfs-rdma-sidecar/Dockerfile.sidecar deleted file mode 100644 index d5f21c7e7..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.sidecar +++ /dev/null @@ -1,55 +0,0 @@ -# Multi-stage build for Go Sidecar -FROM golang:1.26-alpine AS builder - -# Install build dependencies -RUN apk add --no-cache git ca-certificates tzdata - -# Set work directory -WORKDIR /app - -# Copy go mod files -COPY go.mod go.sum ./ - -# Download dependencies -RUN go mod download - -# Copy source code -COPY cmd/ ./cmd/ -COPY pkg/ ./pkg/ - -# Build the binaries -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o demo-server ./cmd/demo-server -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o sidecar ./cmd/sidecar -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o test-rdma ./cmd/test-rdma - -# Runtime stage -FROM alpine:3.18 - -# Install runtime dependencies -RUN apk --no-cache add ca-certificates curl jq - -# Create app user -RUN addgroup -g 1001 appgroup && \ - adduser -D -s /bin/sh -u 1001 -G appgroup appuser - -# Set work directory -WORKDIR /app - -# Copy binaries from builder stage -COPY --from=builder /app/demo-server . -COPY --from=builder /app/sidecar . -COPY --from=builder /app/test-rdma . - -# Change ownership -RUN chown -R appuser:appgroup /app -USER appuser - -# Expose the demo server port -EXPOSE 8081 - -# Health check -HEALTHCHECK --interval=10s --timeout=5s --start-period=15s --retries=3 \ - CMD curl -f http://localhost:8081/health || exit 1 - -# Default command (demo server) -CMD ["./demo-server", "--port", "8081", "--enable-rdma", "--debug"] diff --git a/seaweedfs-rdma-sidecar/Dockerfile.test-client b/seaweedfs-rdma-sidecar/Dockerfile.test-client deleted file mode 100644 index 99d015bcb..000000000 --- a/seaweedfs-rdma-sidecar/Dockerfile.test-client +++ /dev/null @@ -1,59 +0,0 @@ -# Multi-stage build for Test Client -FROM golang:1.26-alpine AS builder - -# Install build dependencies -RUN apk add --no-cache git ca-certificates tzdata - -# Set work directory -WORKDIR /app - -# Copy go mod files -COPY go.mod go.sum ./ - -# Download dependencies -RUN go mod download - -# Copy source code -COPY cmd/ ./cmd/ -COPY pkg/ ./pkg/ - -# Build the test binaries -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o test-rdma ./cmd/test-rdma -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o demo-server ./cmd/demo-server - -# Runtime stage -FROM alpine:3.18 - -# Install runtime dependencies and testing tools -RUN apk --no-cache add \ - ca-certificates \ - curl \ - jq \ - bash \ - wget \ - netcat-openbsd \ - && rm -rf /var/cache/apk/* - -# Create app user -RUN addgroup -g 1001 appgroup && \ - adduser -D -s /bin/bash -u 1001 -G appgroup appuser - -# Set work directory -WORKDIR /app - -# Copy binaries from builder stage -COPY --from=builder /app/test-rdma . -COPY --from=builder /app/demo-server . - -# Copy test scripts -COPY tests/ ./tests/ -RUN chmod +x ./tests/*.sh - -# Change ownership -RUN chown -R appuser:appgroup /app - -# Switch to app user -USER appuser - -# Default command -CMD ["/bin/bash"] diff --git a/seaweedfs-rdma-sidecar/FUTURE-WORK-TODO.md b/seaweedfs-rdma-sidecar/FUTURE-WORK-TODO.md deleted file mode 100644 index cc7457b90..000000000 --- a/seaweedfs-rdma-sidecar/FUTURE-WORK-TODO.md +++ /dev/null @@ -1,276 +0,0 @@ -# SeaweedFS RDMA Sidecar - Future Work TODO - -## ๐ŸŽฏ **Current Status (โœ… COMPLETED)** - -### **Phase 1: Architecture & Integration - DONE** -- โœ… **Complete Go โ†” Rust IPC Pipeline**: Unix sockets + MessagePack -- โœ… **SeaweedFS Integration**: Mount client with RDMA acceleration -- โœ… **Docker Orchestration**: Multi-service setup with proper networking -- โœ… **Error Handling**: Robust fallback and recovery mechanisms -- โœ… **Performance Optimizations**: Zero-copy page cache + connection pooling -- โœ… **Code Quality**: All GitHub PR review comments addressed -- โœ… **Testing Framework**: Integration tests and benchmarking tools - -### **Phase 2: Mock Implementation - DONE** -- โœ… **Mock RDMA Engine**: Complete Rust implementation for development -- โœ… **Pattern Data Generation**: Predictable test data for validation -- โœ… **Simulated Performance**: Realistic latency and throughput modeling -- โœ… **Development Environment**: Full testing without hardware requirements - ---- - -## ๐Ÿš€ **PHASE 3: REAL RDMA IMPLEMENTATION** - -### **3.1 Hardware Abstraction Layer** ๐Ÿ”ด **HIGH PRIORITY** - -#### **Replace Mock RDMA Context** -**File**: `rdma-engine/src/rdma.rs` -**Current**: -```rust -RdmaContextImpl::Mock(MockRdmaContext::new(config).await?) -``` -**TODO**: -```rust -// Enable UCX feature and implement -RdmaContextImpl::Ucx(UcxRdmaContext::new(config).await?) -``` - -**Tasks**: -- [ ] Implement `UcxRdmaContext` struct -- [ ] Add UCX FFI bindings for Rust -- [ ] Handle UCX initialization and cleanup -- [ ] Add feature flag: `real-ucx` vs `mock` - -#### **Real Memory Management** -**File**: `rdma-engine/src/rdma.rs` lines 245-270 -**Current**: Fake memory regions in vector -**TODO**: -- [ ] Integrate with UCX memory registration APIs -- [ ] Implement HugePage support for large transfers -- [ ] Add memory region caching for performance -- [ ] Handle registration/deregistration errors - -#### **Actual RDMA Operations** -**File**: `rdma-engine/src/rdma.rs` lines 273-335 -**Current**: Pattern data + artificial latency -**TODO**: -- [ ] Replace `post_read()` with real UCX RDMA operations -- [ ] Implement `post_write()` with actual memory transfers -- [ ] Add completion polling from hardware queues -- [ ] Handle partial transfers and retries - -### **3.2 Data Path Replacement** ๐ŸŸก **MEDIUM PRIORITY** - -#### **Real Data Transfer** -**File**: `pkg/rdma/client.go` lines 420-442 -**Current**: -```go -// MOCK: Pattern generation -mockData[i] = byte(i % 256) -``` -**TODO**: -```go -// Get actual data from RDMA buffer -realData := getRdmaBufferContents(startResp.LocalAddr, startResp.TransferSize) -validateDataIntegrity(realData, completeResp.ServerCrc) -``` - -**Tasks**: -- [ ] Remove mock data generation -- [ ] Access actual RDMA transferred data -- [ ] Implement CRC validation: `completeResp.ServerCrc` -- [ ] Add data integrity error handling - -#### **Hardware Device Detection** -**File**: `rdma-engine/src/rdma.rs` lines 222-233 -**Current**: Hardcoded Mellanox device info -**TODO**: -- [ ] Enumerate real RDMA devices using UCX -- [ ] Query actual device capabilities -- [ ] Handle multiple device scenarios -- [ ] Add device selection logic - -### **3.3 Performance Optimization** ๐ŸŸข **LOW PRIORITY** - -#### **Memory Registration Caching** -**TODO**: -- [ ] Implement MR (Memory Region) cache -- [ ] Add LRU eviction for memory pressure -- [ ] Optimize for frequently accessed regions -- [ ] Monitor cache hit rates - -#### **Advanced RDMA Features** -**TODO**: -- [ ] Implement RDMA Write operations -- [ ] Add Immediate Data support -- [ ] Implement RDMA Write with Immediate -- [ ] Add Atomic operations (if needed) - -#### **Multi-Transport Support** -**TODO**: -- [ ] Leverage UCX's automatic transport selection -- [ ] Add InfiniBand support -- [ ] Add RoCE (RDMA over Converged Ethernet) support -- [ ] Implement TCP fallback via UCX - ---- - -## ๐Ÿ”ง **PHASE 4: PRODUCTION HARDENING** - -### **4.1 Error Handling & Recovery** -- [ ] Add RDMA-specific error codes -- [ ] Implement connection recovery -- [ ] Add retry logic for transient failures -- [ ] Handle device hot-plug scenarios - -### **4.2 Monitoring & Observability** -- [ ] Add RDMA-specific metrics (bandwidth, latency, errors) -- [ ] Implement tracing for RDMA operations -- [ ] Add health checks for RDMA devices -- [ ] Create performance dashboards - -### **4.3 Configuration & Tuning** -- [ ] Add RDMA-specific configuration options -- [ ] Implement auto-tuning based on workload -- [ ] Add support for multiple RDMA ports -- [ ] Create deployment guides for different hardware - ---- - -## ๐Ÿ“‹ **IMMEDIATE NEXT STEPS** - -### **Step 1: UCX Integration Setup** -1. **Add UCX dependencies to Rust**: - ```toml - [dependencies] - ucx-sys = "0.1" # UCX FFI bindings - ``` - -2. **Create UCX wrapper module**: - ```bash - touch rdma-engine/src/ucx.rs - ``` - -3. **Implement basic UCX context**: - ```rust - pub struct UcxRdmaContext { - context: *mut ucx_sys::ucp_context_h, - worker: *mut ucx_sys::ucp_worker_h, - } - ``` - -### **Step 2: Development Environment** -1. **Install UCX library**: - ```bash - # Ubuntu/Debian - sudo apt-get install libucx-dev - - # CentOS/RHEL - sudo yum install ucx-devel - ``` - -2. **Update Cargo.toml features**: - ```toml - [features] - default = ["mock"] - mock = [] - real-ucx = ["ucx-sys"] - ``` - -### **Step 3: Testing Strategy** -1. **Add hardware detection tests** -2. **Create UCX initialization tests** -3. **Implement gradual feature migration** -4. **Maintain mock fallback for CI/CD** - ---- - -## ๐Ÿ—๏ธ **ARCHITECTURE NOTES** - -### **Current Working Components** -- โœ… **Go Sidecar**: Production-ready HTTP API -- โœ… **IPC Layer**: Robust Unix socket + MessagePack -- โœ… **SeaweedFS Integration**: Complete mount client integration -- โœ… **Docker Setup**: Multi-service orchestration -- โœ… **Error Handling**: Comprehensive fallback mechanisms - -### **Mock vs Real Boundary** -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SeaweedFS โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Go Sidecar โ”‚โ”€โ”€โ”€โ”€โ–ถโ”‚ Rust Engine โ”‚ -โ”‚ (REAL) โ”‚ โ”‚ (REAL) โ”‚ โ”‚ (MOCK) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ - โ–ผ - โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” - โ”‚ RDMA Hardware โ”‚ - โ”‚ (TO IMPLEMENT) โ”‚ - โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### **Performance Expectations** -- **Current Mock**: ~403 ops/sec, 2.48ms latency -- **Target Real**: ~4000 ops/sec, 250ฮผs latency (UCX optimized) -- **Bandwidth Goal**: 25-100 Gbps (depending on hardware) - ---- - -## ๐Ÿ“š **REFERENCE MATERIALS** - -### **UCX Documentation** -- **GitHub**: https://github.com/openucx/ucx -- **API Reference**: https://openucx.readthedocs.io/ -- **Rust Bindings**: https://crates.io/crates/ucx-sys - -### **RDMA Programming** -- **InfiniBand Architecture**: Volume 1 Specification -- **RoCE Standards**: IBTA Annex A17 -- **Performance Tuning**: UCX Performance Guide - -### **SeaweedFS Integration** -- **File ID Format**: `weed/storage/needle/file_id.go` -- **Volume Server**: `weed/server/volume_server_handlers_read.go` -- **Mount Client**: `weed/mount/filehandle_read.go` - ---- - -## โš ๏ธ **IMPORTANT NOTES** - -### **Breaking Changes to Avoid** -- **Keep IPC Protocol Stable**: Don't change MessagePack format -- **Maintain HTTP API**: Existing endpoints must remain compatible -- **Preserve Configuration**: Environment variables should work unchanged - -### **Testing Requirements** -- **Hardware Tests**: Require actual RDMA NICs -- **CI/CD Compatibility**: Must fallback to mock for automated testing -- **Performance Benchmarks**: Compare mock vs real performance - -### **Security Considerations** -- **Memory Protection**: Ensure RDMA regions are properly isolated -- **Access Control**: Validate remote memory access permissions -- **Data Validation**: Always verify CRC checksums - ---- - -## ๐ŸŽฏ **SUCCESS CRITERIA** - -### **Phase 3 Complete When**: -- [ ] Real RDMA data transfers working -- [ ] Hardware device detection functional -- [ ] Performance exceeds mock implementation -- [ ] All integration tests passing with real hardware - -### **Phase 4 Complete When**: -- [ ] Production deployment successful -- [ ] Monitoring and alerting operational -- [ ] Performance targets achieved -- [ ] Error handling validated under load - ---- - -**๐Ÿ“… Last Updated**: December 2024 -**๐Ÿ‘ค Contact**: Resume from `seaweedfs-rdma-sidecar/` directory -**๐Ÿท๏ธ Version**: v1.0 (Mock Implementation Complete) - -**๐Ÿš€ Ready to resume**: All infrastructure is in place, just need to replace the mock RDMA layer with UCX integration! diff --git a/seaweedfs-rdma-sidecar/Makefile b/seaweedfs-rdma-sidecar/Makefile deleted file mode 100644 index 19aa90461..000000000 --- a/seaweedfs-rdma-sidecar/Makefile +++ /dev/null @@ -1,205 +0,0 @@ -# SeaweedFS RDMA Sidecar Makefile - -.PHONY: help build test clean docker-build docker-test docker-clean integration-test - -# Default target -help: ## Show this help message - @echo "SeaweedFS RDMA Sidecar - Available Commands:" - @echo "" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-20s\033[0m %s\n", $$1, $$2}' - @echo "" - @echo "Examples:" - @echo " make build # Build all components locally" - @echo " make docker-test # Run complete Docker integration tests" - @echo " make test # Run unit tests" - -# Local Build Targets -build: build-go build-rust ## Build all components locally - -build-go: ## Build Go components (sidecar, demo-server, test-rdma) - @echo "๐Ÿ”จ Building Go components..." - go build -o bin/sidecar ./cmd/sidecar - go build -o bin/demo-server ./cmd/demo-server - go build -o bin/test-rdma ./cmd/test-rdma - @echo "โœ… Go build complete" - -build-rust: ## Build Rust RDMA engine - @echo "๐Ÿฆ€ Building Rust RDMA engine..." - cd rdma-engine && cargo build --release - @echo "โœ… Rust build complete" - -# Testing Targets -test: test-go test-rust ## Run all unit tests - -test-go: ## Run Go tests - @echo "๐Ÿงช Running Go tests..." - go test ./... - @echo "โœ… Go tests complete" - -test-rust: ## Run Rust tests - @echo "๐Ÿงช Running Rust tests..." - cd rdma-engine && cargo test - @echo "โœ… Rust tests complete" - -integration-test: build ## Run local integration test - @echo "๐Ÿ”— Running local integration test..." - ./scripts/demo-e2e.sh - @echo "โœ… Local integration test complete" - -# Docker Targets -docker-build: ## Build all Docker images - @echo "๐Ÿณ Building Docker images..." - docker-compose build - @echo "โœ… Docker images built" - -docker-start: ## Start Docker services - @echo "๐Ÿš€ Starting Docker services..." - ./tests/docker-test-helper.sh start - @echo "โœ… Docker services started" - -docker-test: ## Run Docker integration tests - @echo "๐Ÿงช Running Docker integration tests..." - ./tests/docker-test-helper.sh test - @echo "โœ… Docker integration tests complete" - -docker-stop: ## Stop Docker services - @echo "๐Ÿ›‘ Stopping Docker services..." - ./tests/docker-test-helper.sh stop - @echo "โœ… Docker services stopped" - -docker-clean: ## Clean Docker services and volumes - @echo "๐Ÿงน Cleaning Docker environment..." - ./tests/docker-test-helper.sh clean - docker system prune -f - @echo "โœ… Docker cleanup complete" - -docker-logs: ## Show Docker logs - ./tests/docker-test-helper.sh logs - -docker-status: ## Show Docker service status - ./tests/docker-test-helper.sh status - -docker-shell: ## Open interactive shell in test container - ./tests/docker-test-helper.sh shell - -# RDMA Simulation Targets -rdma-sim-build: ## Build RDMA simulation environment - @echo "๐Ÿš€ Building RDMA simulation environment..." - docker-compose -f docker-compose.rdma-sim.yml build - @echo "โœ… RDMA simulation images built" - -rdma-sim-start: ## Start RDMA simulation environment - @echo "๐Ÿš€ Starting RDMA simulation environment..." - docker-compose -f docker-compose.rdma-sim.yml up -d - @echo "โœ… RDMA simulation environment started" - -rdma-sim-test: ## Run RDMA simulation tests - @echo "๐Ÿงช Running RDMA simulation tests..." - docker-compose -f docker-compose.rdma-sim.yml run --rm integration-tests-rdma - @echo "โœ… RDMA simulation tests complete" - -rdma-sim-stop: ## Stop RDMA simulation environment - @echo "๐Ÿ›‘ Stopping RDMA simulation environment..." - docker-compose -f docker-compose.rdma-sim.yml down - @echo "โœ… RDMA simulation environment stopped" - -rdma-sim-clean: ## Clean RDMA simulation environment - @echo "๐Ÿงน Cleaning RDMA simulation environment..." - docker-compose -f docker-compose.rdma-sim.yml down -v --remove-orphans - docker system prune -f - @echo "โœ… RDMA simulation cleanup complete" - -rdma-sim-status: ## Check RDMA simulation status - @echo "๐Ÿ“Š RDMA simulation status:" - docker-compose -f docker-compose.rdma-sim.yml ps - @echo "" - @echo "๐Ÿ” RDMA device status:" - docker-compose -f docker-compose.rdma-sim.yml exec rdma-simulation /opt/rdma-sim/test-rdma.sh || true - -rdma-sim-shell: ## Open shell in RDMA simulation container - @echo "๐Ÿš Opening RDMA simulation shell..." - docker-compose -f docker-compose.rdma-sim.yml exec rdma-simulation /bin/bash - -rdma-sim-logs: ## Show RDMA simulation logs - docker-compose -f docker-compose.rdma-sim.yml logs - -rdma-sim-ucx: ## Show UCX information in simulation - @echo "๐Ÿ“‹ UCX information in simulation:" - docker-compose -f docker-compose.rdma-sim.yml exec rdma-simulation /opt/rdma-sim/ucx-info.sh - -# Development Targets -dev-setup: ## Set up development environment - @echo "๐Ÿ› ๏ธ Setting up development environment..." - go mod tidy - cd rdma-engine && cargo check - chmod +x scripts/*.sh tests/*.sh - @echo "โœ… Development environment ready" - -format: ## Format code - @echo "โœจ Formatting code..." - go fmt ./... - cd rdma-engine && cargo fmt - @echo "โœ… Code formatted" - -lint: ## Run linters - @echo "๐Ÿ” Running linters..." - go vet ./... - cd rdma-engine && cargo clippy -- -D warnings - @echo "โœ… Linting complete" - -# Cleanup Targets -clean: clean-go clean-rust ## Clean all build artifacts - -clean-go: ## Clean Go build artifacts - @echo "๐Ÿงน Cleaning Go artifacts..." - rm -rf bin/ - go clean -testcache - @echo "โœ… Go artifacts cleaned" - -clean-rust: ## Clean Rust build artifacts - @echo "๐Ÿงน Cleaning Rust artifacts..." - cd rdma-engine && cargo clean - @echo "โœ… Rust artifacts cleaned" - -# Full Workflow Targets -check: format lint test ## Format, lint, and test everything - -ci: check integration-test docker-test ## Complete CI workflow - -demo: build ## Run local demo - @echo "๐ŸŽฎ Starting local demo..." - ./scripts/demo-e2e.sh - -# Docker Development Workflow -docker-dev: docker-clean docker-build docker-test ## Complete Docker development cycle - -# Quick targets -quick-test: build ## Quick local test - ./bin/test-rdma --help - -quick-docker: ## Quick Docker test - docker-compose up -d rdma-engine rdma-sidecar - sleep 5 - curl -s http://localhost:8081/health | jq '.' - docker-compose down - -# Help and Documentation -docs: ## Generate/update documentation - @echo "๐Ÿ“š Documentation ready:" - @echo " README.md - Main project documentation" - @echo " DOCKER-TESTING.md - Docker integration testing guide" - @echo " Use 'make help' for available commands" - -# Environment Info -info: ## Show environment information - @echo "๐Ÿ” Environment Information:" - @echo " Go Version: $$(go version)" - @echo " Rust Version: $$(cd rdma-engine && cargo --version)" - @echo " Docker Version: $$(docker --version)" - @echo " Docker Compose Version: $$(docker-compose --version)" - @echo "" - @echo "๐Ÿ—๏ธ Project Structure:" - @echo " Go Components: cmd/ pkg/" - @echo " Rust Engine: rdma-engine/" - @echo " Tests: tests/" - @echo " Scripts: scripts/" diff --git a/seaweedfs-rdma-sidecar/README.md b/seaweedfs-rdma-sidecar/README.md deleted file mode 100644 index 559da79de..000000000 --- a/seaweedfs-rdma-sidecar/README.md +++ /dev/null @@ -1,385 +0,0 @@ -# ๐Ÿš€ SeaweedFS RDMA Sidecar - -**High-Performance RDMA Acceleration for SeaweedFS using UCX and Rust** - -[![Build Status](https://img.shields.io/badge/build-passing-brightgreen)](#) -[![Go Version](https://img.shields.io/badge/go-1.23+-blue)](#) -[![Rust Version](https://img.shields.io/badge/rust-1.70+-orange)](#) -[![License](https://img.shields.io/badge/license-MIT-green)](#) - -## ๐ŸŽฏ Overview - -This project implements a **high-performance RDMA (Remote Direct Memory Access) sidecar** for SeaweedFS that provides significant performance improvements for data-intensive read operations. The sidecar uses a **hybrid Go + Rust architecture** with the [UCX (Unified Communication X)](https://github.com/openucx/ucx) framework to deliver up to **44x performance improvement** over traditional HTTP-based reads. - -### ๐Ÿ—๏ธ Architecture - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ SeaweedFS โ”‚ โ”‚ Go Sidecar โ”‚ โ”‚ Rust Engine โ”‚ -โ”‚ Volume Server โ”‚โ—„โ”€โ”€โ–บโ”‚ (Control Plane) โ”‚โ—„โ”€โ”€โ–บโ”‚ (Data Plane) โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ - โ”‚ โ”‚ โ”‚ - โ”‚ โ”‚ โ”‚ - โ–ผ โ–ผ โ–ผ - HTTP/gRPC API RDMA Client API UCX/RDMA Hardware -``` - -**Components:** -- **๐ŸŸข Go Sidecar**: Control plane handling SeaweedFS integration, client API, and fallback logic -- **๐Ÿฆ€ Rust Engine**: High-performance data plane with UCX framework for RDMA operations -- **๐Ÿ”— IPC Bridge**: Unix domain socket communication with MessagePack serialization - -## ๐ŸŒŸ Key Features - -### โšก Performance -- **44x faster** than HTTP reads (theoretical max based on RDMA vs TCP overhead) -- **Sub-microsecond latency** for memory-mapped operations -- **Zero-copy data transfers** directly to/from SeaweedFS volume files -- **Concurrent session management** with up to 1000+ simultaneous operations - -### ๐Ÿ›ก๏ธ Reliability -- **Automatic HTTP fallback** when RDMA unavailable -- **Graceful degradation** under failure conditions -- **Session timeout and cleanup** to prevent resource leaks -- **Comprehensive error handling** with structured logging - -### ๐Ÿ”ง Production Ready -- **Container-native deployment** with Kubernetes support -- **RDMA device plugin integration** for hardware resource management -- **HugePages optimization** for memory efficiency -- **Prometheus metrics** and structured logging for observability - -### ๐ŸŽš๏ธ Flexibility -- **Mock RDMA implementation** for development and testing -- **Configurable transport selection** (RDMA, TCP, shared memory via UCX) -- **Multi-device support** with automatic failover -- **Authentication and authorization** support - -## ๐Ÿš€ Quick Start - -### Prerequisites - -```bash -# Required dependencies -- Go 1.26+ -- Rust 1.70+ -- UCX libraries (for hardware RDMA) -- Linux with RDMA-capable hardware (InfiniBand/RoCE) - -# Optional for development -- Docker -- Kubernetes -- jq (for demo scripts) -``` - -### ๐Ÿ—๏ธ Build - -```bash -# Clone the repository -git clone -cd seaweedfs-rdma-sidecar - -# Build Go components -go build -o bin/sidecar ./cmd/sidecar -go build -o bin/test-rdma ./cmd/test-rdma -go build -o bin/demo-server ./cmd/demo-server - -# Build Rust engine -cd rdma-engine -cargo build --release -cd .. -``` - -### ๐ŸŽฎ Demo - -Run the complete end-to-end demonstration: - -```bash -# Interactive demo with all components -./scripts/demo-e2e.sh - -# Or run individual components -./rdma-engine/target/release/rdma-engine-server --debug & -./bin/demo-server --port 8080 --enable-rdma -``` - -## ๐Ÿ“Š Performance Results - -### Mock RDMA Performance (Development) -``` -Average Latency: 2.48ms per operation -Throughput: 403.2 operations/sec -Success Rate: 100% -Session Management: โœ… Working -IPC Communication: โœ… Working -``` - -### Expected Hardware RDMA Performance -``` -Average Latency: < 10ยตs per operation (440x improvement) -Throughput: > 1M operations/sec (2500x improvement) -Bandwidth: > 100 Gbps (theoretical InfiniBand limit) -CPU Utilization: < 5% (vs 60%+ for HTTP) -``` - -## ๐Ÿงฉ Components - -### 1๏ธโƒฃ Rust RDMA Engine (`rdma-engine/`) - -High-performance data plane built with: - -- **๐Ÿ”ง UCX Integration**: Production-grade RDMA framework -- **โšก Async Operations**: Tokio-based async runtime -- **๐Ÿง  Memory Management**: Pooled buffers with HugePage support -- **๐Ÿ“ก IPC Server**: Unix domain socket with MessagePack -- **๐Ÿ“Š Session Management**: Thread-safe lifecycle handling - -```rust -// Example: Starting the RDMA engine -let config = RdmaEngineConfig { - device_name: "auto".to_string(), - port: 18515, - max_sessions: 1000, - // ... other config -}; - -let engine = RdmaEngine::new(config).await?; -engine.start().await?; -``` - -### 2๏ธโƒฃ Go Sidecar (`pkg/`, `cmd/`) - -Control plane providing: - -- **๐Ÿ”Œ SeaweedFS Integration**: Native needle read/write support -- **๐Ÿ”„ HTTP Fallback**: Automatic degradation when RDMA unavailable -- **๐Ÿ“ˆ Performance Monitoring**: Metrics and benchmarking -- **๐ŸŒ HTTP API**: RESTful interface for management - -```go -// Example: Using the RDMA client -client := seaweedfs.NewSeaweedFSRDMAClient(&seaweedfs.Config{ - RDMASocketPath: "/tmp/rdma-engine.sock", - Enabled: true, -}) - -resp, err := client.ReadNeedle(ctx, &seaweedfs.NeedleReadRequest{ - VolumeID: 1, - NeedleID: 12345, - Size: 4096, -}) -``` - -### 3๏ธโƒฃ Integration Examples (`cmd/demo-server/`) - -Production-ready integration examples: - -- **๐ŸŒ HTTP Server**: Demonstrates SeaweedFS integration -- **๐Ÿ“Š Benchmarking**: Performance testing utilities -- **๐Ÿ” Health Checks**: Monitoring and diagnostics -- **๐Ÿ“ฑ Web Interface**: Browser-based demo and testing - -## ๐Ÿณ Deployment - -### Kubernetes - -```yaml -apiVersion: v1 -kind: Pod -metadata: - name: seaweedfs-with-rdma -spec: - containers: - - name: volume-server - image: chrislusf/seaweedfs:latest - # ... volume server config - - - name: rdma-sidecar - image: seaweedfs-rdma-sidecar:latest - resources: - limits: - rdma/hca: 1 # RDMA device - hugepages-2Mi: 1Gi - volumeMounts: - - name: rdma-socket - mountPath: /tmp/rdma-engine.sock -``` - -### Docker Compose - -```yaml -version: '3.8' -services: - rdma-engine: - build: - context: . - dockerfile: rdma-engine/Dockerfile - privileged: true - volumes: - - /tmp/rdma-engine.sock:/tmp/rdma-engine.sock - - seaweedfs-sidecar: - build: . - depends_on: - - rdma-engine - ports: - - "8080:8080" - volumes: - - /tmp/rdma-engine.sock:/tmp/rdma-engine.sock -``` - -## ๐Ÿงช Testing - -### Unit Tests -```bash -# Go tests -go test ./... - -# Rust tests -cd rdma-engine && cargo test -``` - -### Integration Tests -```bash -# Full end-to-end testing -./scripts/demo-e2e.sh - -# Direct RDMA engine testing -./bin/test-rdma ping -./bin/test-rdma capabilities -./bin/test-rdma read --volume 1 --needle 12345 -./bin/test-rdma bench --iterations 100 -``` - -### Performance Benchmarking -```bash -# HTTP vs RDMA comparison -./bin/demo-server --enable-rdma & -curl "http://localhost:8080/benchmark?iterations=1000&size=1048576" -``` - -## ๐Ÿ”ง Configuration - -### RDMA Engine Configuration - -```toml -# rdma-engine/config.toml -[rdma] -device_name = "mlx5_0" # or "auto" -port = 18515 -max_sessions = 1000 -buffer_size = "1GB" - -[ipc] -socket_path = "/tmp/rdma-engine.sock" -max_connections = 100 - -[logging] -level = "info" -``` - -### Go Sidecar Configuration - -```yaml -# config.yaml -rdma: - socket_path: "/tmp/rdma-engine.sock" - enabled: true - timeout: "30s" - -seaweedfs: - volume_server_url: "http://localhost:8080" - -http: - port: 8080 - enable_cors: true -``` - -## ๐Ÿ“ˆ Monitoring - -### Metrics - -The sidecar exposes Prometheus-compatible metrics: - -- `rdma_operations_total{type="read|write", result="success|error"}` -- `rdma_operation_duration_seconds{type="read|write"}` -- `rdma_sessions_active` -- `rdma_bytes_transferred_total{direction="tx|rx"}` - -### Health Checks - -```bash -# Sidecar health -curl http://localhost:8080/health - -# RDMA engine health -curl http://localhost:8080/stats -``` - -### Logging - -Structured logging with configurable levels: - -```json -{ - "timestamp": "2025-08-16T20:55:17Z", - "level": "INFO", - "message": "โœ… RDMA read completed successfully", - "session_id": "db152578-bfad-4cb3-a50f-a2ac66eecc6a", - "bytes_read": 1024, - "duration": "2.48ms", - "transfer_rate": 800742.88 -} -``` - -## ๐Ÿ› ๏ธ Development - -### Mock RDMA Mode - -For development without RDMA hardware: - -```bash -# Enable mock mode (default) -cargo run --features mock-ucx - -# All operations simulate RDMA with realistic latencies -``` - -### UCX Hardware Mode - -For production with real RDMA hardware: - -```bash -# Enable hardware UCX -cargo run --features real-ucx - -# Requires UCX libraries and RDMA-capable hardware -``` - -### Adding New Operations - -1. **Define protobuf messages** in `rdma-engine/src/ipc.rs` -2. **Implement Go client** in `pkg/ipc/client.go` -3. **Add Rust handler** in `rdma-engine/src/ipc.rs` -4. **Update tests** in both languages - -## ๐Ÿ™ Acknowledgments - -- **[UCX Project](https://github.com/openucx/ucx)** - Unified Communication X framework -- **[SeaweedFS](https://github.com/seaweedfs/seaweedfs)** - Distributed file system -- **Rust Community** - Excellent async/await and FFI capabilities -- **Go Community** - Robust networking and gRPC libraries - -## ๐Ÿ“ž Support - -- ๐Ÿ› **Bug Reports**: [Create an issue](../../issues/new?template=bug_report.md) -- ๐Ÿ’ก **Feature Requests**: [Create an issue](../../issues/new?template=feature_request.md) -- ๐Ÿ“š **Documentation**: See [docs/](docs/) folder -- ๐Ÿ’ฌ **Discussions**: [GitHub Discussions](../../discussions) - ---- - -**๐Ÿš€ Ready to accelerate your SeaweedFS deployment with RDMA?** - -Get started with the [Quick Start Guide](#-quick-start) or explore the [Demo Server](cmd/demo-server/) for hands-on experience! - diff --git a/seaweedfs-rdma-sidecar/REVIEW_FEEDBACK.md b/seaweedfs-rdma-sidecar/REVIEW_FEEDBACK.md deleted file mode 100644 index 5034f1bf0..000000000 --- a/seaweedfs-rdma-sidecar/REVIEW_FEEDBACK.md +++ /dev/null @@ -1,55 +0,0 @@ -# PR #7140 Review Feedback Summary - -## Positive Feedback Received โœ… - -### Source: [GitHub PR #7140 Review](https://github.com/seaweedfs/seaweedfs/pull/7140#pullrequestreview-3126580539) -**Reviewer**: Gemini Code Assist (Automated Review Bot) -**Date**: August 18, 2025 - -## Comments Analysis - -### ๐Ÿ† Binary Search Optimization - PRAISED -**File**: `weed/mount/filehandle_read.go` -**Implementation**: Efficient chunk lookup using binary search with cached cumulative offsets - -**Reviewer Comment**: -> "The `tryRDMARead` function efficiently finds the target chunk for a given offset by using a binary search on cached cumulative chunk offsets. This is an effective optimization that will perform well even for files with a large number of chunks." - -**Technical Merit**: -- โœ… O(log N) performance vs O(N) linear search -- โœ… Cached cumulative offsets prevent repeated calculations -- โœ… Scales well for large fragmented files -- โœ… Memory-efficient implementation - -### ๐Ÿ† Resource Management - PRAISED -**File**: `weed/mount/weedfs.go` -**Implementation**: Proper RDMA client initialization and cleanup - -**Reviewer Comment**: -> "The RDMA client is now correctly initialized and attached to the `WFS` struct when RDMA is enabled. The shutdown logic in the `grace.OnInterrupt` handler has also been updated to properly close the RDMA client, preventing resource leaks." - -**Technical Merit**: -- โœ… Proper initialization with error handling -- โœ… Clean shutdown in interrupt handler -- โœ… No resource leaks -- โœ… Graceful degradation on failure - -## Summary - -**All review comments are positive acknowledgments of excellent implementation practices.** - -### Key Strengths Recognized: -1. **Performance Optimization**: Binary search algorithm implementation -2. **Memory Safety**: Proper resource lifecycle management -3. **Code Quality**: Clean, efficient, and maintainable code -4. **Production Readiness**: Robust error handling and cleanup - -### Build Status: โœ… PASSING -- โœ… `go build ./...` - All packages compile successfully -- โœ… `go vet ./...` - No linting issues -- โœ… All tests passing -- โœ… Docker builds working - -## Conclusion - -The RDMA sidecar implementation has received positive feedback from automated code review, confirming high code quality and adherence to best practices. **No action items required** - these are endorsements of excellent work. diff --git a/seaweedfs-rdma-sidecar/WEED-MOUNT-CODE-PATH.md b/seaweedfs-rdma-sidecar/WEED-MOUNT-CODE-PATH.md deleted file mode 100644 index 1fdace934..000000000 --- a/seaweedfs-rdma-sidecar/WEED-MOUNT-CODE-PATH.md +++ /dev/null @@ -1,260 +0,0 @@ -# ๐Ÿ“‹ Weed Mount RDMA Integration - Code Path Analysis - -## Current Status - -The RDMA client (`RDMAMountClient`) exists in `weed/mount/rdma_client.go` but is **not yet integrated** into the actual file read path. The integration points are identified but not implemented. - -## ๐Ÿ” Complete Code Path - -### **1. FUSE Read Request Entry Point** -```go -// File: weed/mount/weedfs_file_read.go:41 -func (wfs *WFS) Read(cancel <-chan struct{}, in *fuse.ReadIn, buff []byte) (fuse.ReadResult, fuse.Status) { - fh := wfs.GetHandle(FileHandleId(in.Fh)) - // ... - offset := int64(in.Offset) - totalRead, err := readDataByFileHandleWithContext(ctx, buff, fh, offset) - // ... - return fuse.ReadResultData(buff[:totalRead]), fuse.OK -} -``` - -### **2. File Handle Read Coordination** -```go -// File: weed/mount/weedfs_file_read.go:103 -func readDataByFileHandleWithContext(ctx context.Context, buff []byte, fhIn *FileHandle, offset int64) (int64, error) { - size := len(buff) - fhIn.lockForRead(offset, size) - defer fhIn.unlockForRead(offset, size) - - // KEY INTEGRATION POINT: This is where RDMA should be attempted - n, tsNs, err := fhIn.readFromChunksWithContext(ctx, buff, offset) - // ... - return n, err -} -``` - -### **3. Chunk Reading (Current Implementation)** -```go -// File: weed/mount/filehandle_read.go:29 -func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte, offset int64) (int64, int64, error) { - // ... - - // CURRENT: Direct chunk reading without RDMA - totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(ctx, fileSize, buff, offset) - - // MISSING: RDMA integration should happen here - return int64(totalRead), ts, err -} -``` - -### **4. RDMA Integration Point (What Needs to Be Added)** - -The integration should happen in `readFromChunksWithContext` like this: - -```go -func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte, offset int64) (int64, int64, error) { - // ... existing code ... - - // NEW: Try RDMA acceleration first - if fh.wfs.rdmaClient != nil && fh.wfs.rdmaClient.IsHealthy() { - if totalRead, ts, err := fh.tryRDMARead(ctx, buff, offset); err == nil { - glog.V(4).Infof("RDMA read successful: %d bytes", totalRead) - return totalRead, ts, nil - } - glog.V(2).Infof("RDMA read failed, falling back to HTTP") - } - - // FALLBACK: Original HTTP-based chunk reading - totalRead, ts, err := fh.entryChunkGroup.ReadDataAt(ctx, fileSize, buff, offset) - return int64(totalRead), ts, err -} -``` - -## ๐Ÿš€ RDMA Client Integration - -### **5. RDMA Read Implementation (Already Exists)** -```go -// File: weed/mount/rdma_client.go:129 -func (c *RDMAMountClient) ReadNeedle(ctx context.Context, volumeID uint32, needleID uint64, cookie uint32, offset, size uint64) ([]byte, bool, error) { - // Prepare request URL - reqURL := fmt.Sprintf("http://%s/read?volume=%d&needle=%d&cookie=%d&offset=%d&size=%d", - c.sidecarAddr, volumeID, needleID, cookie, offset, size) - - // Execute HTTP request to RDMA sidecar - resp, err := c.httpClient.Do(req) - // ... - - // Return data with RDMA metadata - return data, isRDMA, nil -} -``` - -### **6. RDMA Sidecar Processing** -```go -// File: seaweedfs-rdma-sidecar/cmd/demo-server/main.go:375 -func (s *DemoServer) readHandler(w http.ResponseWriter, r *http.Request) { - // Parse volume, needle, cookie from URL parameters - volumeID, _ := strconv.ParseUint(query.Get("volume"), 10, 32) - needleID, _ := strconv.ParseUint(query.Get("needle"), 10, 64) - - // Use distributed client for volume lookup + RDMA - if s.useDistributed && s.distributedClient != nil { - resp, err = s.distributedClient.ReadNeedle(ctx, req) - } else { - resp, err = s.rdmaClient.ReadNeedle(ctx, req) // Local RDMA - } - - // Return binary data or JSON metadata - w.Write(resp.Data) -} -``` - -### **7. Volume Lookup & RDMA Engine** -```go -// File: seaweedfs-rdma-sidecar/pkg/seaweedfs/distributed_client.go:45 -func (c *DistributedRDMAClient) ReadNeedle(ctx context.Context, req *NeedleReadRequest) (*NeedleReadResponse, error) { - // Step 1: Lookup volume location from master - locations, err := c.locationService.LookupVolume(ctx, req.VolumeID) - - // Step 2: Find best server (local preferred) - bestLocation := c.locationService.FindBestLocation(locations) - - // Step 3: Make HTTP request to target server's RDMA sidecar - return c.makeRDMARequest(ctx, req, bestLocation, start) -} -``` - -### **8. Rust RDMA Engine (Final Data Access)** -```rust -// File: rdma-engine/src/ipc.rs:403 -async fn handle_start_read(req: StartReadRequest, ...) -> RdmaResult { - // Create RDMA session - let session_id = Uuid::new_v4().to_string(); - let buffer = vec![0u8; transfer_size as usize]; - - // Register memory for RDMA - let memory_region = rdma_context.register_memory(local_addr, transfer_size).await?; - - // Perform RDMA read (mock implementation) - rdma_context.post_read(local_addr, remote_addr, remote_key, size, wr_id).await?; - let completions = rdma_context.poll_completion(1).await?; - - // Return session info - Ok(StartReadResponse { session_id, local_addr, ... }) -} -``` - -## ๐Ÿ”ง Missing Integration Components - -### **1. WFS Struct Extension** -```go -// File: weed/mount/weedfs.go (needs modification) -type WFS struct { - // ... existing fields ... - rdmaClient *RDMAMountClient // ADD THIS -} -``` - -### **2. RDMA Client Initialization** -```go -// File: weed/command/mount.go (needs modification) -func runMount(cmd *cobra.Command, args []string) bool { - // ... existing code ... - - // NEW: Initialize RDMA client if enabled - var rdmaClient *mount.RDMAMountClient - if *mountOptions.rdmaEnabled && *mountOptions.rdmaSidecarAddr != "" { - rdmaClient, err = mount.NewRDMAMountClient( - *mountOptions.rdmaSidecarAddr, - *mountOptions.rdmaMaxConcurrent, - *mountOptions.rdmaTimeoutMs, - ) - if err != nil { - glog.Warningf("Failed to initialize RDMA client: %v", err) - } - } - - // Pass RDMA client to WFS - wfs := mount.NewSeaweedFileSystem(&mount.Option{ - // ... existing options ... - RDMAClient: rdmaClient, // ADD THIS - }) -} -``` - -### **3. Chunk-to-Needle Mapping** -```go -// File: weed/mount/filehandle_read.go (needs new method) -func (fh *FileHandle) tryRDMARead(ctx context.Context, buff []byte, offset int64) (int64, int64, error) { - entry := fh.GetEntry() - - // Find which chunk contains the requested offset - for _, chunk := range entry.GetEntry().Chunks { - if offset >= chunk.Offset && offset < chunk.Offset+int64(chunk.Size) { - // Parse chunk.FileId to get volume, needle, cookie - volumeID, needleID, cookie, err := ParseFileId(chunk.FileId) - if err != nil { - return 0, 0, err - } - - // Calculate offset within the chunk - chunkOffset := uint64(offset - chunk.Offset) - readSize := uint64(min(len(buff), int(chunk.Size-chunkOffset))) - - // Make RDMA request - data, isRDMA, err := fh.wfs.rdmaClient.ReadNeedle( - ctx, volumeID, needleID, cookie, chunkOffset, readSize) - if err != nil { - return 0, 0, err - } - - // Copy data to buffer - copied := copy(buff, data) - return int64(copied), time.Now().UnixNano(), nil - } - } - - return 0, 0, fmt.Errorf("chunk not found for offset %d", offset) -} -``` - -## ๐Ÿ“Š Request Flow Summary - -1. **User Application** โ†’ `read()` system call -2. **FUSE Kernel** โ†’ Routes to `WFS.Read()` -3. **WFS.Read()** โ†’ Calls `readDataByFileHandleWithContext()` -4. **readDataByFileHandleWithContext()** โ†’ Calls `fh.readFromChunksWithContext()` -5. **readFromChunksWithContext()** โ†’ **[INTEGRATION POINT]** Try RDMA first -6. **tryRDMARead()** โ†’ Parse chunk info, call `RDMAMountClient.ReadNeedle()` -7. **RDMAMountClient** โ†’ HTTP request to RDMA sidecar -8. **RDMA Sidecar** โ†’ Volume lookup + RDMA engine call -9. **RDMA Engine** โ†’ Direct memory access via RDMA hardware -10. **Response Path** โ†’ Data flows back through all layers to user - -## โœ… What's Working vs Missing - -### **โœ… Already Implemented:** -- โœ… `RDMAMountClient` with HTTP communication -- โœ… RDMA sidecar with volume lookup -- โœ… Rust RDMA engine with mock hardware -- โœ… File ID parsing utilities -- โœ… Health checks and statistics -- โœ… Command-line flags for RDMA options - -### **โŒ Missing Integration:** -- โŒ RDMA client not added to WFS struct -- โŒ RDMA client not initialized in mount command -- โŒ `tryRDMARead()` method not implemented -- โŒ Chunk-to-needle mapping logic missing -- โŒ RDMA integration not wired into read path - -## ๐ŸŽฏ Next Steps - -1. **Add RDMA client to WFS struct and Option** -2. **Initialize RDMA client in mount command** -3. **Implement `tryRDMARead()` method** -4. **Wire RDMA integration into `readFromChunksWithContext()`** -5. **Test end-to-end RDMA acceleration** - -The architecture is sound and most components exist - only the final integration wiring is needed! diff --git a/seaweedfs-rdma-sidecar/cmd/demo-server/main.go b/seaweedfs-rdma-sidecar/cmd/demo-server/main.go deleted file mode 100644 index 42b5020e5..000000000 --- a/seaweedfs-rdma-sidecar/cmd/demo-server/main.go +++ /dev/null @@ -1,663 +0,0 @@ -// Package main provides a demonstration server showing SeaweedFS RDMA integration -package main - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "os/signal" - "strconv" - "strings" - "syscall" - "time" - - "seaweedfs-rdma-sidecar/pkg/seaweedfs" - - "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var ( - port int - rdmaSocket string - volumeServerURL string - enableRDMA bool - enableZeroCopy bool - tempDir string - enablePooling bool - maxConnections int - maxIdleTime time.Duration - debug bool -) - -func main() { - var rootCmd = &cobra.Command{ - Use: "demo-server", - Short: "SeaweedFS RDMA integration demonstration server", - Long: `Demonstration server that shows how SeaweedFS can integrate with the RDMA sidecar -for accelerated read operations. This server provides HTTP endpoints that demonstrate -the RDMA fast path with HTTP fallback capabilities.`, - RunE: runServer, - } - - rootCmd.Flags().IntVarP(&port, "port", "p", 8080, "Demo server HTTP port") - rootCmd.Flags().StringVarP(&rdmaSocket, "rdma-socket", "r", "/tmp/rdma-engine.sock", "Path to RDMA engine Unix socket") - rootCmd.Flags().StringVarP(&volumeServerURL, "volume-server", "v", "http://localhost:8080", "SeaweedFS volume server URL for HTTP fallback") - rootCmd.Flags().BoolVarP(&enableRDMA, "enable-rdma", "e", true, "Enable RDMA acceleration") - rootCmd.Flags().BoolVarP(&enableZeroCopy, "enable-zerocopy", "z", true, "Enable zero-copy optimization via temp files") - rootCmd.Flags().StringVarP(&tempDir, "temp-dir", "t", "/tmp/rdma-cache", "Temp directory for zero-copy files") - rootCmd.Flags().BoolVar(&enablePooling, "enable-pooling", true, "Enable RDMA connection pooling") - rootCmd.Flags().IntVar(&maxConnections, "max-connections", 10, "Maximum connections in RDMA pool") - rootCmd.Flags().DurationVar(&maxIdleTime, "max-idle-time", 5*time.Minute, "Maximum idle time for pooled connections") - rootCmd.Flags().BoolVarP(&debug, "debug", "d", false, "Enable debug logging") - - if err := rootCmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func runServer(cmd *cobra.Command, args []string) error { - // Setup logging - logger := logrus.New() - if debug { - logger.SetLevel(logrus.DebugLevel) - logger.SetFormatter(&logrus.TextFormatter{ - FullTimestamp: true, - ForceColors: true, - }) - } else { - logger.SetLevel(logrus.InfoLevel) - } - - logger.WithFields(logrus.Fields{ - "port": port, - "rdma_socket": rdmaSocket, - "volume_server_url": volumeServerURL, - "enable_rdma": enableRDMA, - "enable_zerocopy": enableZeroCopy, - "temp_dir": tempDir, - "enable_pooling": enablePooling, - "max_connections": maxConnections, - "max_idle_time": maxIdleTime, - "debug": debug, - }).Info("๐Ÿš€ Starting SeaweedFS RDMA Demo Server") - - // Create SeaweedFS RDMA client - config := &seaweedfs.Config{ - RDMASocketPath: rdmaSocket, - VolumeServerURL: volumeServerURL, - Enabled: enableRDMA, - DefaultTimeout: 30 * time.Second, - Logger: logger, - TempDir: tempDir, - UseZeroCopy: enableZeroCopy, - EnablePooling: enablePooling, - MaxConnections: maxConnections, - MaxIdleTime: maxIdleTime, - } - - rdmaClient, err := seaweedfs.NewSeaweedFSRDMAClient(config) - if err != nil { - return fmt.Errorf("failed to create RDMA client: %w", err) - } - - // Start RDMA client - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - if err := rdmaClient.Start(ctx); err != nil { - logger.WithError(err).Error("Failed to start RDMA client") - } - cancel() - - // Create demo server - server := &DemoServer{ - rdmaClient: rdmaClient, - logger: logger, - } - - // Setup HTTP routes - mux := http.NewServeMux() - mux.HandleFunc("/", server.homeHandler) - mux.HandleFunc("/health", server.healthHandler) - mux.HandleFunc("/stats", server.statsHandler) - mux.HandleFunc("/read", server.readHandler) - mux.HandleFunc("/benchmark", server.benchmarkHandler) - mux.HandleFunc("/cleanup", server.cleanupHandler) - - httpServer := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - Handler: mux, - } - - // Handle graceful shutdown - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go func() { - logger.WithField("port", port).Info("๐ŸŒ Demo server starting") - if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed { - logger.WithError(err).Fatal("HTTP server failed") - } - }() - - // Wait for shutdown signal - <-sigChan - logger.Info("๐Ÿ“ก Received shutdown signal, gracefully shutting down...") - - // Shutdown HTTP server - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer shutdownCancel() - - if err := httpServer.Shutdown(shutdownCtx); err != nil { - logger.WithError(err).Error("HTTP server shutdown failed") - } else { - logger.Info("๐ŸŒ HTTP server shutdown complete") - } - - // Stop RDMA client - rdmaClient.Stop() - logger.Info("๐Ÿ›‘ Demo server shutdown complete") - - return nil -} - -// DemoServer demonstrates SeaweedFS RDMA integration -type DemoServer struct { - rdmaClient *seaweedfs.SeaweedFSRDMAClient - logger *logrus.Logger -} - -// homeHandler provides information about the demo server -func (s *DemoServer) homeHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - w.Header().Set("Content-Type", "text/html") - fmt.Fprintf(w, ` - - - SeaweedFS RDMA Demo Server - - - -
-

๐Ÿš€ SeaweedFS RDMA Demo Server

-

This server demonstrates SeaweedFS integration with RDMA acceleration for high-performance reads.

- -
- RDMA Status: %s -
- -

๐Ÿ“‹ Available Endpoints

- -
-

๐Ÿฅ Health Check

-

/health - Check server and RDMA engine health

-
- -
-

๐Ÿ“Š Statistics

-

/stats - Get RDMA client statistics and capabilities

-
- -
-

๐Ÿ“– Read Needle

-

/read - Read a needle with RDMA fast path

-

Parameters: file_id OR (volume, needle, cookie), volume_server, offset (optional), size (optional)

-
- -
-

๐Ÿ Benchmark

-

/benchmark - Run performance benchmark

-

Parameters: iterations (default: 10), size (default: 4096)

-
- -

๐Ÿ“ Example Usage

- 
-# Read a needle using file ID (recommended)
-curl "http://localhost:%d/read?file_id=3,01637037d6&size=1024&volume_server=http://localhost:8080"
-
-# Read a needle using individual parameters (legacy)
-curl "http://localhost:%d/read?volume=1&needle=12345&cookie=305419896&size=1024&volume_server=http://localhost:8080"
-
-# Read a needle (hex cookie)
-curl "http://localhost:%d/read?volume=1&needle=12345&cookie=0x12345678&size=1024&volume_server=http://localhost:8080"
-
-# Run benchmark
-curl "http://localhost:%d/benchmark?iterations=5&size=2048"
-
-# Check health
-curl "http://localhost:%d/health"
-
-
- -`, - map[bool]string{true: "enabled", false: "disabled"}[s.rdmaClient.IsEnabled()], - map[bool]string{true: "RDMA Enabled โœ…", false: "RDMA Disabled (HTTP Fallback Only) โš ๏ธ"}[s.rdmaClient.IsEnabled()], - port, port, port, port) -} - -// healthHandler checks server and RDMA health -func (s *DemoServer) healthHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - health := map[string]interface{}{ - "status": "healthy", - "timestamp": time.Now().Format(time.RFC3339), - "rdma": map[string]interface{}{ - "enabled": false, - "connected": false, - }, - } - - if s.rdmaClient != nil { - health["rdma"].(map[string]interface{})["enabled"] = s.rdmaClient.IsEnabled() - health["rdma"].(map[string]interface{})["type"] = "local" - - if s.rdmaClient.IsEnabled() { - if err := s.rdmaClient.HealthCheck(ctx); err != nil { - s.logger.WithError(err).Warn("RDMA health check failed") - health["rdma"].(map[string]interface{})["error"] = err.Error() - } else { - health["rdma"].(map[string]interface{})["connected"] = true - } - } - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(health) -} - -// statsHandler returns RDMA statistics -func (s *DemoServer) statsHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - var stats map[string]interface{} - - if s.rdmaClient != nil { - stats = s.rdmaClient.GetStats() - stats["client_type"] = "local" - } else { - stats = map[string]interface{}{ - "client_type": "none", - "error": "no RDMA client available", - } - } - - stats["timestamp"] = time.Now().Format(time.RFC3339) - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(stats) -} - -// readHandler demonstrates needle reading with RDMA -func (s *DemoServer) readHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse parameters - support both file_id and individual parameters for backward compatibility - query := r.URL.Query() - volumeServer := query.Get("volume_server") - fileID := query.Get("file_id") - - var volumeID, cookie uint64 - var needleID uint64 - var err error - - if fileID != "" { - // Use file ID format (e.g., "3,01637037d6") - // Extract individual components using existing SeaweedFS parsing - fid, parseErr := needle.ParseFileIdFromString(fileID) - if parseErr != nil { - http.Error(w, fmt.Sprintf("invalid 'file_id' parameter: %v", parseErr), http.StatusBadRequest) - return - } - volumeID = uint64(fid.VolumeId) - needleID = uint64(fid.Key) - cookie = uint64(fid.Cookie) - } else { - // Use individual parameters (backward compatibility) - volumeID, err = strconv.ParseUint(query.Get("volume"), 10, 32) - if err != nil { - http.Error(w, "invalid 'volume' parameter", http.StatusBadRequest) - return - } - - needleID, err = strconv.ParseUint(query.Get("needle"), 10, 64) - if err != nil { - http.Error(w, "invalid 'needle' parameter", http.StatusBadRequest) - return - } - - // Parse cookie parameter - support both decimal and hexadecimal formats - cookieStr := query.Get("cookie") - if strings.HasPrefix(strings.ToLower(cookieStr), "0x") { - // Parse as hexadecimal (remove "0x" prefix) - cookie, err = strconv.ParseUint(cookieStr[2:], 16, 32) - } else { - // Parse as decimal (default) - cookie, err = strconv.ParseUint(cookieStr, 10, 32) - } - if err != nil { - http.Error(w, "invalid 'cookie' parameter (expected decimal or hex with 0x prefix)", http.StatusBadRequest) - return - } - } - - var offset uint64 - if offsetStr := query.Get("offset"); offsetStr != "" { - var parseErr error - offset, parseErr = strconv.ParseUint(offsetStr, 10, 64) - if parseErr != nil { - http.Error(w, "invalid 'offset' parameter", http.StatusBadRequest) - return - } - } - - var size uint64 - if sizeStr := query.Get("size"); sizeStr != "" { - var parseErr error - size, parseErr = strconv.ParseUint(sizeStr, 10, 64) - if parseErr != nil { - http.Error(w, "invalid 'size' parameter", http.StatusBadRequest) - return - } - } - - if volumeServer == "" { - http.Error(w, "volume_server parameter is required", http.StatusBadRequest) - return - } - - if volumeID == 0 || needleID == 0 { - http.Error(w, "volume and needle parameters are required", http.StatusBadRequest) - return - } - - // Note: cookie and size can have defaults for demo purposes when user provides empty values, - // but invalid parsing is caught above with proper error responses - if cookie == 0 { - cookie = 0x12345678 // Default cookie for demo - } - - if size == 0 { - size = 4096 // Default size - } - - logFields := logrus.Fields{ - "volume_server": volumeServer, - "volume_id": volumeID, - "needle_id": needleID, - "cookie": fmt.Sprintf("0x%x", cookie), - "offset": offset, - "size": size, - } - if fileID != "" { - logFields["file_id"] = fileID - } - s.logger.WithFields(logFields).Info("๐Ÿ“– Processing needle read request") - - ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) - defer cancel() - - start := time.Now() - req := &seaweedfs.NeedleReadRequest{ - VolumeID: uint32(volumeID), - NeedleID: needleID, - Cookie: uint32(cookie), - Offset: offset, - Size: size, - VolumeServer: volumeServer, - } - - resp, err := s.rdmaClient.ReadNeedle(ctx, req) - - if err != nil { - s.logger.WithError(err).Error("โŒ Needle read failed") - http.Error(w, fmt.Sprintf("Read failed: %v", err), http.StatusInternalServerError) - return - } - - duration := time.Since(start) - - s.logger.WithFields(logrus.Fields{ - "volume_id": volumeID, - "needle_id": needleID, - "is_rdma": resp.IsRDMA, - "source": resp.Source, - "duration": duration, - "data_size": len(resp.Data), - }).Info("โœ… Needle read completed") - - // Return metadata and first few bytes - result := map[string]interface{}{ - "success": true, - "volume_id": volumeID, - "needle_id": needleID, - "cookie": fmt.Sprintf("0x%x", cookie), - "is_rdma": resp.IsRDMA, - "source": resp.Source, - "session_id": resp.SessionID, - "duration": duration.String(), - "data_size": len(resp.Data), - "timestamp": time.Now().Format(time.RFC3339), - "use_temp_file": resp.UseTempFile, - "temp_file": resp.TempFilePath, - } - - // Set headers for zero-copy optimization - if resp.UseTempFile && resp.TempFilePath != "" { - w.Header().Set("X-Use-Temp-File", "true") - w.Header().Set("X-Temp-File", resp.TempFilePath) - w.Header().Set("X-Source", resp.Source) - w.Header().Set("X-RDMA-Used", fmt.Sprintf("%t", resp.IsRDMA)) - - // For zero-copy, return minimal JSON response and let client read from temp file - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) - return - } - - // Regular response with data - w.Header().Set("X-Source", resp.Source) - w.Header().Set("X-RDMA-Used", fmt.Sprintf("%t", resp.IsRDMA)) - - // Include first 32 bytes as hex for verification - if len(resp.Data) > 0 { - displayLen := 32 - if len(resp.Data) < displayLen { - displayLen = len(resp.Data) - } - result["data_preview"] = fmt.Sprintf("%x", resp.Data[:displayLen]) - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) -} - -// benchmarkHandler runs performance benchmarks -func (s *DemoServer) benchmarkHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse parameters - query := r.URL.Query() - - iterations := 10 // default value - if iterationsStr := query.Get("iterations"); iterationsStr != "" { - var parseErr error - iterations, parseErr = strconv.Atoi(iterationsStr) - if parseErr != nil { - http.Error(w, "invalid 'iterations' parameter", http.StatusBadRequest) - return - } - } - - size := uint64(4096) // default value - if sizeStr := query.Get("size"); sizeStr != "" { - var parseErr error - size, parseErr = strconv.ParseUint(sizeStr, 10, 64) - if parseErr != nil { - http.Error(w, "invalid 'size' parameter", http.StatusBadRequest) - return - } - } - - if iterations <= 0 { - iterations = 10 - } - if size == 0 { - size = 4096 - } - - s.logger.WithFields(logrus.Fields{ - "iterations": iterations, - "size": size, - }).Info("๐Ÿ Starting benchmark") - - ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) - defer cancel() - - var rdmaSuccessful, rdmaFailed, httpSuccessful, httpFailed int - var totalDuration time.Duration - var totalBytes uint64 - - startTime := time.Now() - - for i := 0; i < iterations; i++ { - req := &seaweedfs.NeedleReadRequest{ - VolumeID: 1, - NeedleID: uint64(i + 1), - Cookie: 0x12345678, - Offset: 0, - Size: size, - } - - opStart := time.Now() - resp, err := s.rdmaClient.ReadNeedle(ctx, req) - opDuration := time.Since(opStart) - - if err != nil { - httpFailed++ - continue - } - - totalDuration += opDuration - totalBytes += uint64(len(resp.Data)) - - if resp.IsRDMA { - rdmaSuccessful++ - } else { - httpSuccessful++ - } - } - - benchDuration := time.Since(startTime) - - // Calculate statistics - totalOperations := rdmaSuccessful + httpSuccessful - avgLatency := time.Duration(0) - if totalOperations > 0 { - avgLatency = totalDuration / time.Duration(totalOperations) - } - - throughputMBps := float64(totalBytes) / benchDuration.Seconds() / (1024 * 1024) - opsPerSec := float64(totalOperations) / benchDuration.Seconds() - - result := map[string]interface{}{ - "benchmark_results": map[string]interface{}{ - "iterations": iterations, - "size_per_op": size, - "total_duration": benchDuration.String(), - "successful_ops": totalOperations, - "failed_ops": rdmaFailed + httpFailed, - "rdma_ops": rdmaSuccessful, - "http_ops": httpSuccessful, - "avg_latency": avgLatency.String(), - "throughput_mbps": fmt.Sprintf("%.2f", throughputMBps), - "ops_per_sec": fmt.Sprintf("%.1f", opsPerSec), - "total_bytes": totalBytes, - }, - "rdma_enabled": s.rdmaClient.IsEnabled(), - "timestamp": time.Now().Format(time.RFC3339), - } - - s.logger.WithFields(logrus.Fields{ - "iterations": iterations, - "successful_ops": totalOperations, - "rdma_ops": rdmaSuccessful, - "http_ops": httpSuccessful, - "avg_latency": avgLatency, - "throughput_mbps": throughputMBps, - "ops_per_sec": opsPerSec, - }).Info("๐Ÿ“Š Benchmark completed") - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) -} - -// cleanupHandler handles temp file cleanup requests from mount clients -func (s *DemoServer) cleanupHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodDelete { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Get temp file path from query parameters - tempFilePath := r.URL.Query().Get("temp_file") - if tempFilePath == "" { - http.Error(w, "missing 'temp_file' parameter", http.StatusBadRequest) - return - } - - s.logger.WithField("temp_file", tempFilePath).Debug("๐Ÿ—‘๏ธ Processing cleanup request") - - // Use the RDMA client's cleanup method (which delegates to seaweedfs client) - err := s.rdmaClient.CleanupTempFile(tempFilePath) - if err != nil { - s.logger.WithError(err).WithField("temp_file", tempFilePath).Warn("Failed to cleanup temp file") - http.Error(w, fmt.Sprintf("cleanup failed: %v", err), http.StatusInternalServerError) - return - } - - s.logger.WithField("temp_file", tempFilePath).Debug("๐Ÿงน Temp file cleanup successful") - - // Return success response - w.Header().Set("Content-Type", "application/json") - response := map[string]interface{}{ - "success": true, - "message": "temp file cleaned up successfully", - "temp_file": tempFilePath, - "timestamp": time.Now().Format(time.RFC3339), - } - json.NewEncoder(w).Encode(response) -} diff --git a/seaweedfs-rdma-sidecar/cmd/sidecar/main.go b/seaweedfs-rdma-sidecar/cmd/sidecar/main.go deleted file mode 100644 index 55d98c4c6..000000000 --- a/seaweedfs-rdma-sidecar/cmd/sidecar/main.go +++ /dev/null @@ -1,345 +0,0 @@ -// Package main provides the main RDMA sidecar service that integrates with SeaweedFS -package main - -import ( - "context" - "encoding/json" - "fmt" - "net/http" - "os" - "os/signal" - "strconv" - "syscall" - "time" - - "seaweedfs-rdma-sidecar/pkg/rdma" - - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var ( - port int - engineSocket string - debug bool - timeout time.Duration -) - -// Response structs for JSON encoding -type HealthResponse struct { - Status string `json:"status"` - RdmaEngineConnected bool `json:"rdma_engine_connected"` - RdmaEngineLatency string `json:"rdma_engine_latency"` - Timestamp string `json:"timestamp"` -} - -type CapabilitiesResponse struct { - Version string `json:"version"` - DeviceName string `json:"device_name"` - VendorId uint32 `json:"vendor_id"` - MaxSessions uint32 `json:"max_sessions"` - MaxTransferSize uint64 `json:"max_transfer_size"` - ActiveSessions uint32 `json:"active_sessions"` - RealRdma bool `json:"real_rdma"` - PortGid string `json:"port_gid"` - PortLid uint16 `json:"port_lid"` - SupportedAuth []string `json:"supported_auth"` -} - -type PingResponse struct { - Success bool `json:"success"` - EngineLatency string `json:"engine_latency"` - TotalLatency string `json:"total_latency"` - Timestamp string `json:"timestamp"` -} - -func main() { - var rootCmd = &cobra.Command{ - Use: "rdma-sidecar", - Short: "SeaweedFS RDMA acceleration sidecar", - Long: `RDMA sidecar that accelerates SeaweedFS read/write operations using UCX and Rust RDMA engine. - -This sidecar acts as a bridge between SeaweedFS volume servers and the high-performance -Rust RDMA engine, providing significant performance improvements for data-intensive workloads.`, - RunE: runSidecar, - } - - // Flags - rootCmd.Flags().IntVarP(&port, "port", "p", 8081, "HTTP server port") - rootCmd.Flags().StringVarP(&engineSocket, "engine-socket", "e", "/tmp/rdma-engine.sock", "Path to RDMA engine Unix socket") - rootCmd.Flags().BoolVarP(&debug, "debug", "d", false, "Enable debug logging") - rootCmd.Flags().DurationVarP(&timeout, "timeout", "t", 30*time.Second, "RDMA operation timeout") - - if err := rootCmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func runSidecar(cmd *cobra.Command, args []string) error { - // Setup logging - logger := logrus.New() - if debug { - logger.SetLevel(logrus.DebugLevel) - logger.SetFormatter(&logrus.TextFormatter{ - FullTimestamp: true, - ForceColors: true, - }) - } else { - logger.SetLevel(logrus.InfoLevel) - } - - logger.WithFields(logrus.Fields{ - "port": port, - "engine_socket": engineSocket, - "debug": debug, - "timeout": timeout, - }).Info("๐Ÿš€ Starting SeaweedFS RDMA Sidecar") - - // Create RDMA client - rdmaConfig := &rdma.Config{ - EngineSocketPath: engineSocket, - DefaultTimeout: timeout, - Logger: logger, - } - - rdmaClient := rdma.NewClient(rdmaConfig) - - // Connect to RDMA engine - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - logger.Info("๐Ÿ”— Connecting to RDMA engine...") - if err := rdmaClient.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect to RDMA engine: %w", err) - } - logger.Info("โœ… Connected to RDMA engine successfully") - - // Create HTTP server - sidecar := &Sidecar{ - rdmaClient: rdmaClient, - logger: logger, - } - - mux := http.NewServeMux() - - // Health check endpoint - mux.HandleFunc("/health", sidecar.healthHandler) - - // RDMA operations endpoints - mux.HandleFunc("/rdma/read", sidecar.rdmaReadHandler) - mux.HandleFunc("/rdma/capabilities", sidecar.capabilitiesHandler) - mux.HandleFunc("/rdma/ping", sidecar.pingHandler) - - server := &http.Server{ - Addr: fmt.Sprintf(":%d", port), - Handler: mux, - } - - // Handle graceful shutdown - sigChan := make(chan os.Signal, 1) - signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM) - - go func() { - logger.WithField("port", port).Info("๐ŸŒ HTTP server starting") - if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { - logger.WithError(err).Fatal("HTTP server failed") - } - }() - - // Wait for shutdown signal - <-sigChan - logger.Info("๐Ÿ“ก Received shutdown signal, gracefully shutting down...") - - // Shutdown HTTP server - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 10*time.Second) - defer shutdownCancel() - - if err := server.Shutdown(shutdownCtx); err != nil { - logger.WithError(err).Error("HTTP server shutdown failed") - } else { - logger.Info("๐ŸŒ HTTP server shutdown complete") - } - - // Disconnect from RDMA engine - rdmaClient.Disconnect() - logger.Info("๐Ÿ›‘ RDMA sidecar shutdown complete") - - return nil -} - -// Sidecar represents the main sidecar service -type Sidecar struct { - rdmaClient *rdma.Client - logger *logrus.Logger -} - -// Health check handler -func (s *Sidecar) healthHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - defer cancel() - - // Test RDMA engine connectivity - if !s.rdmaClient.IsConnected() { - s.logger.Warn("โš ๏ธ RDMA engine not connected") - http.Error(w, "RDMA engine not connected", http.StatusServiceUnavailable) - return - } - - // Ping RDMA engine - latency, err := s.rdmaClient.Ping(ctx) - if err != nil { - s.logger.WithError(err).Error("โŒ RDMA engine ping failed") - http.Error(w, "RDMA engine ping failed", http.StatusServiceUnavailable) - return - } - - w.Header().Set("Content-Type", "application/json") - response := HealthResponse{ - Status: "healthy", - RdmaEngineConnected: true, - RdmaEngineLatency: latency.String(), - Timestamp: time.Now().Format(time.RFC3339), - } - json.NewEncoder(w).Encode(response) -} - -// RDMA capabilities handler -func (s *Sidecar) capabilitiesHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - caps := s.rdmaClient.GetCapabilities() - if caps == nil { - http.Error(w, "No capabilities available", http.StatusServiceUnavailable) - return - } - - w.Header().Set("Content-Type", "application/json") - response := CapabilitiesResponse{ - Version: caps.Version, - DeviceName: caps.DeviceName, - VendorId: caps.VendorId, - MaxSessions: uint32(caps.MaxSessions), - MaxTransferSize: caps.MaxTransferSize, - ActiveSessions: uint32(caps.ActiveSessions), - RealRdma: caps.RealRdma, - PortGid: caps.PortGid, - PortLid: caps.PortLid, - SupportedAuth: caps.SupportedAuth, - } - json.NewEncoder(w).Encode(response) -} - -// RDMA ping handler -func (s *Sidecar) pingHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) - defer cancel() - - start := time.Now() - latency, err := s.rdmaClient.Ping(ctx) - totalLatency := time.Since(start) - - if err != nil { - s.logger.WithError(err).Error("โŒ RDMA ping failed") - http.Error(w, fmt.Sprintf("Ping failed: %v", err), http.StatusInternalServerError) - return - } - - w.Header().Set("Content-Type", "application/json") - response := PingResponse{ - Success: true, - EngineLatency: latency.String(), - TotalLatency: totalLatency.String(), - Timestamp: time.Now().Format(time.RFC3339), - } - json.NewEncoder(w).Encode(response) -} - -// RDMA read handler - uses GET method with query parameters for RESTful read operations -func (s *Sidecar) rdmaReadHandler(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodGet { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - // Parse query parameters - query := r.URL.Query() - - // Get file ID (e.g., "3,01637037d6") - this is the natural SeaweedFS identifier - fileID := query.Get("file_id") - if fileID == "" { - http.Error(w, "missing 'file_id' parameter", http.StatusBadRequest) - return - } - - // Parse optional offset and size parameters - offset := uint64(0) // default value - if offsetStr := query.Get("offset"); offsetStr != "" { - val, err := strconv.ParseUint(offsetStr, 10, 64) - if err != nil { - http.Error(w, "invalid 'offset' parameter", http.StatusBadRequest) - return - } - offset = val - } - - size := uint64(4096) // default value - if sizeStr := query.Get("size"); sizeStr != "" { - val, err := strconv.ParseUint(sizeStr, 10, 64) - if err != nil { - http.Error(w, "invalid 'size' parameter", http.StatusBadRequest) - return - } - size = val - } - - s.logger.WithFields(logrus.Fields{ - "file_id": fileID, - "offset": offset, - "size": size, - }).Info("๐Ÿ“– Processing RDMA read request") - - ctx, cancel := context.WithTimeout(r.Context(), timeout) - defer cancel() - - start := time.Now() - resp, err := s.rdmaClient.ReadFileRange(ctx, fileID, offset, size) - duration := time.Since(start) - - if err != nil { - s.logger.WithError(err).Error("โŒ RDMA read failed") - http.Error(w, fmt.Sprintf("RDMA read failed: %v", err), http.StatusInternalServerError) - return - } - - s.logger.WithFields(logrus.Fields{ - "file_id": fileID, - "bytes_read": resp.BytesRead, - "duration": duration, - "transfer_rate": resp.TransferRate, - "session_id": resp.SessionID, - }).Info("โœ… RDMA read completed successfully") - - // Set response headers - w.Header().Set("Content-Type", "application/octet-stream") - w.Header().Set("X-RDMA-Session-ID", resp.SessionID) - w.Header().Set("X-RDMA-Duration", duration.String()) - w.Header().Set("X-RDMA-Transfer-Rate", fmt.Sprintf("%.2f", resp.TransferRate)) - w.Header().Set("X-RDMA-Bytes-Read", fmt.Sprintf("%d", resp.BytesRead)) - - // Write the data - w.Write(resp.Data) -} diff --git a/seaweedfs-rdma-sidecar/cmd/test-rdma/main.go b/seaweedfs-rdma-sidecar/cmd/test-rdma/main.go deleted file mode 100644 index 4f2b2da43..000000000 --- a/seaweedfs-rdma-sidecar/cmd/test-rdma/main.go +++ /dev/null @@ -1,295 +0,0 @@ -// Package main provides a test client for the RDMA engine integration -package main - -import ( - "context" - "fmt" - "os" - "time" - - "seaweedfs-rdma-sidecar/pkg/rdma" - - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var ( - socketPath string - debug bool - timeout time.Duration - volumeID uint32 - needleID uint64 - cookie uint32 - offset uint64 - size uint64 -) - -func main() { - var rootCmd = &cobra.Command{ - Use: "test-rdma", - Short: "Test client for SeaweedFS RDMA engine integration", - Long: `Test client that demonstrates communication between Go sidecar and Rust RDMA engine. - -This tool allows you to test various RDMA operations including: -- Engine connectivity and capabilities -- RDMA read operations with mock data -- Performance measurements -- IPC protocol validation`, - } - - // Global flags - defaultSocketPath := os.Getenv("RDMA_SOCKET_PATH") - if defaultSocketPath == "" { - defaultSocketPath = "/tmp/rdma-engine.sock" - } - rootCmd.PersistentFlags().StringVarP(&socketPath, "socket", "s", defaultSocketPath, "Path to RDMA engine Unix socket (env: RDMA_SOCKET_PATH)") - rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", false, "Enable debug logging") - rootCmd.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Second, "Operation timeout") - - // Subcommands - rootCmd.AddCommand(pingCmd()) - rootCmd.AddCommand(capsCmd()) - rootCmd.AddCommand(readCmd()) - rootCmd.AddCommand(benchCmd()) - - if err := rootCmd.Execute(); err != nil { - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - os.Exit(1) - } -} - -func pingCmd() *cobra.Command { - return &cobra.Command{ - Use: "ping", - Short: "Test connectivity to RDMA engine", - Long: "Send a ping message to the RDMA engine and measure latency", - RunE: func(cmd *cobra.Command, args []string) error { - client := createClient() - defer client.Disconnect() - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - fmt.Printf("๐Ÿ“ Pinging RDMA engine at %s...\n", socketPath) - - if err := client.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - - latency, err := client.Ping(ctx) - if err != nil { - return fmt.Errorf("ping failed: %w", err) - } - - fmt.Printf("โœ… Ping successful! Latency: %v\n", latency) - return nil - }, - } -} - -func capsCmd() *cobra.Command { - return &cobra.Command{ - Use: "capabilities", - Short: "Get RDMA engine capabilities", - Long: "Query the RDMA engine for its current capabilities and status", - RunE: func(cmd *cobra.Command, args []string) error { - client := createClient() - defer client.Disconnect() - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - fmt.Printf("๐Ÿ” Querying RDMA engine capabilities...\n") - - if err := client.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - - caps := client.GetCapabilities() - if caps == nil { - return fmt.Errorf("no capabilities received") - } - - fmt.Printf("\n๐Ÿ“Š RDMA Engine Capabilities:\n") - fmt.Printf(" Version: %s\n", caps.Version) - fmt.Printf(" Max Sessions: %d\n", caps.MaxSessions) - fmt.Printf(" Max Transfer Size: %d bytes (%.1f MB)\n", caps.MaxTransferSize, float64(caps.MaxTransferSize)/(1024*1024)) - fmt.Printf(" Active Sessions: %d\n", caps.ActiveSessions) - fmt.Printf(" Real RDMA: %t\n", caps.RealRdma) - fmt.Printf(" Port GID: %s\n", caps.PortGid) - fmt.Printf(" Port LID: %d\n", caps.PortLid) - fmt.Printf(" Supported Auth: %v\n", caps.SupportedAuth) - - if caps.RealRdma { - fmt.Printf("๐Ÿš€ Hardware RDMA enabled!\n") - } else { - fmt.Printf("๐ŸŸก Using mock RDMA (development mode)\n") - } - - return nil - }, - } -} - -func readCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "read", - Short: "Test RDMA read operation", - Long: "Perform a test RDMA read operation with specified parameters", - RunE: func(cmd *cobra.Command, args []string) error { - client := createClient() - defer client.Disconnect() - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - fmt.Printf("๐Ÿ“– Testing RDMA read operation...\n") - fmt.Printf(" Volume ID: %d\n", volumeID) - fmt.Printf(" Needle ID: %d\n", needleID) - fmt.Printf(" Cookie: 0x%x\n", cookie) - fmt.Printf(" Offset: %d\n", offset) - fmt.Printf(" Size: %d bytes\n", size) - - if err := client.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - - start := time.Now() - resp, err := client.ReadRange(ctx, volumeID, needleID, cookie, offset, size) - if err != nil { - return fmt.Errorf("read failed: %w", err) - } - - duration := time.Since(start) - - fmt.Printf("\nโœ… RDMA read completed successfully!\n") - fmt.Printf(" Session ID: %s\n", resp.SessionID) - fmt.Printf(" Bytes Read: %d\n", resp.BytesRead) - fmt.Printf(" Duration: %v\n", duration) - fmt.Printf(" Transfer Rate: %.2f MB/s\n", resp.TransferRate) - fmt.Printf(" Success: %t\n", resp.Success) - fmt.Printf(" Message: %s\n", resp.Message) - - // Show first few bytes of data for verification - if len(resp.Data) > 0 { - displayLen := 32 - if len(resp.Data) < displayLen { - displayLen = len(resp.Data) - } - fmt.Printf(" Data (first %d bytes): %x\n", displayLen, resp.Data[:displayLen]) - } - - return nil - }, - } - - cmd.Flags().Uint32VarP(&volumeID, "volume", "v", 1, "Volume ID") - cmd.Flags().Uint64VarP(&needleID, "needle", "n", 100, "Needle ID") - cmd.Flags().Uint32VarP(&cookie, "cookie", "c", 0x12345678, "Needle cookie") - cmd.Flags().Uint64VarP(&offset, "offset", "o", 0, "Read offset") - cmd.Flags().Uint64VarP(&size, "size", "z", 4096, "Read size in bytes") - - return cmd -} - -func benchCmd() *cobra.Command { - var ( - iterations int - readSize uint64 - ) - - cmd := &cobra.Command{ - Use: "bench", - Short: "Benchmark RDMA read performance", - Long: "Run multiple RDMA read operations and measure performance statistics", - RunE: func(cmd *cobra.Command, args []string) error { - client := createClient() - defer client.Disconnect() - - ctx, cancel := context.WithTimeout(context.Background(), timeout) - defer cancel() - - fmt.Printf("๐Ÿ Starting RDMA read benchmark...\n") - fmt.Printf(" Iterations: %d\n", iterations) - fmt.Printf(" Read Size: %d bytes\n", readSize) - fmt.Printf(" Socket: %s\n", socketPath) - - if err := client.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect: %w", err) - } - - // Warmup - fmt.Printf("๐Ÿ”ฅ Warming up...\n") - for i := 0; i < 5; i++ { - _, err := client.ReadRange(ctx, 1, uint64(i+1), 0x12345678, 0, readSize) - if err != nil { - return fmt.Errorf("warmup read %d failed: %w", i+1, err) - } - } - - // Benchmark - fmt.Printf("๐Ÿ“Š Running benchmark...\n") - var totalDuration time.Duration - var totalBytes uint64 - successful := 0 - - startTime := time.Now() - for i := 0; i < iterations; i++ { - opStart := time.Now() - resp, err := client.ReadRange(ctx, 1, uint64(i+1), 0x12345678, 0, readSize) - opDuration := time.Since(opStart) - - if err != nil { - fmt.Printf("โŒ Read %d failed: %v\n", i+1, err) - continue - } - - totalDuration += opDuration - totalBytes += resp.BytesRead - successful++ - - if (i+1)%10 == 0 || i == iterations-1 { - fmt.Printf(" Completed %d/%d reads\n", i+1, iterations) - } - } - benchDuration := time.Since(startTime) - - // Calculate statistics - avgLatency := totalDuration / time.Duration(successful) - throughputMBps := float64(totalBytes) / benchDuration.Seconds() / (1024 * 1024) - opsPerSec := float64(successful) / benchDuration.Seconds() - - fmt.Printf("\n๐Ÿ“ˆ Benchmark Results:\n") - fmt.Printf(" Total Duration: %v\n", benchDuration) - fmt.Printf(" Successful Operations: %d/%d (%.1f%%)\n", successful, iterations, float64(successful)/float64(iterations)*100) - fmt.Printf(" Total Bytes Transferred: %d (%.1f MB)\n", totalBytes, float64(totalBytes)/(1024*1024)) - fmt.Printf(" Average Latency: %v\n", avgLatency) - fmt.Printf(" Throughput: %.2f MB/s\n", throughputMBps) - fmt.Printf(" Operations/sec: %.1f\n", opsPerSec) - - return nil - }, - } - - cmd.Flags().IntVarP(&iterations, "iterations", "i", 100, "Number of read operations") - cmd.Flags().Uint64VarP(&readSize, "read-size", "r", 4096, "Size of each read in bytes") - - return cmd -} - -func createClient() *rdma.Client { - logger := logrus.New() - if debug { - logger.SetLevel(logrus.DebugLevel) - } else { - logger.SetLevel(logrus.InfoLevel) - } - - config := &rdma.Config{ - EngineSocketPath: socketPath, - DefaultTimeout: timeout, - Logger: logger, - } - - return rdma.NewClient(config) -} diff --git a/seaweedfs-rdma-sidecar/demo-server b/seaweedfs-rdma-sidecar/demo-server deleted file mode 100755 index 737f1721c..000000000 Binary files a/seaweedfs-rdma-sidecar/demo-server and /dev/null differ diff --git a/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml b/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml deleted file mode 100644 index 14ea1ca78..000000000 --- a/seaweedfs-rdma-sidecar/docker-compose.mount-rdma.yml +++ /dev/null @@ -1,267 +0,0 @@ -services: - # SeaweedFS Master - seaweedfs-master: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-master - ports: - - "9333:9333" - - "19333:19333" - command: > - master - -port=9333 - -mdir=/data - -volumeSizeLimitMB=1024 - -defaultReplication=000 - volumes: - - seaweedfs_master_data:/data - networks: - - seaweedfs-rdma - healthcheck: - test: ["CMD", "wget", "--timeout=10", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:9333/cluster/status"] - interval: 10s - timeout: 10s - retries: 6 - start_period: 60s - - # SeaweedFS Volume Server - seaweedfs-volume: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-volume - ports: - - "8080:8080" - - "18080:18080" - command: > - volume - -master=seaweedfs-master:9333 - -port=8080 - -dir=/data - -max=100 - volumes: - - seaweedfs_volume_data:/data - networks: - - seaweedfs-rdma - depends_on: - seaweedfs-master: - condition: service_healthy - healthcheck: - test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8080"] - interval: 10s - timeout: 10s - retries: 6 - start_period: 30s - - # SeaweedFS Filer - seaweedfs-filer: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-filer - ports: - - "8888:8888" - - "18888:18888" - command: > - filer - -master=seaweedfs-master:9333 - -port=8888 - -defaultReplicaPlacement=000 - networks: - - seaweedfs-rdma - depends_on: - seaweedfs-master: - condition: service_healthy - seaweedfs-volume: - condition: service_healthy - healthcheck: - test: ["CMD", "sh", "-c", "pgrep weed && netstat -tln | grep :8888"] - interval: 10s - timeout: 10s - retries: 6 - start_period: 45s - - # RDMA Engine (Rust) - rdma-engine: - build: - context: . - dockerfile: Dockerfile.rdma-engine - container_name: rdma-engine - volumes: - - rdma_socket:/tmp/rdma - networks: - - seaweedfs-rdma - environment: - - RUST_LOG=debug - - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock - - RDMA_DEVICE=auto - - RDMA_PORT=18515 - - RDMA_GID_INDEX=0 - - DEBUG=true - command: > - ./rdma-engine-server - --ipc-socket ${RDMA_SOCKET_PATH} - --device ${RDMA_DEVICE} - --port ${RDMA_PORT} - --debug - healthcheck: - test: ["CMD", "sh", "-c", "pgrep rdma-engine-server >/dev/null && test -S /tmp/rdma/rdma-engine.sock"] - interval: 5s - timeout: 3s - retries: 5 - start_period: 10s - - # RDMA Sidecar (Go) - rdma-sidecar: - build: - context: . - dockerfile: Dockerfile.sidecar - container_name: rdma-sidecar - ports: - - "8081:8081" - volumes: - - rdma_socket:/tmp/rdma - networks: - - seaweedfs-rdma - environment: - - RDMA_SOCKET_PATH=/tmp/rdma/rdma-engine.sock - - VOLUME_SERVER_URL=http://seaweedfs-volume:8080 - - SIDECAR_PORT=8081 - - ENABLE_RDMA=true - - ENABLE_ZEROCOPY=true - - ENABLE_POOLING=true - - MAX_CONNECTIONS=10 - - MAX_IDLE_TIME=5m - - DEBUG=true - command: > - ./demo-server - --port ${SIDECAR_PORT} - --rdma-socket ${RDMA_SOCKET_PATH} - --volume-server ${VOLUME_SERVER_URL} - --enable-rdma - --enable-zerocopy - --enable-pooling - --max-connections ${MAX_CONNECTIONS} - --max-idle-time ${MAX_IDLE_TIME} - --debug - depends_on: - rdma-engine: - condition: service_healthy - seaweedfs-volume: - condition: service_healthy - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - - # SeaweedFS Mount with RDMA - seaweedfs-mount: - build: - context: . - dockerfile: Dockerfile.mount-rdma - platform: linux/amd64 - container_name: seaweedfs-mount - privileged: true # Required for FUSE - devices: - - /dev/fuse:/dev/fuse - cap_add: - - SYS_ADMIN - volumes: - - seaweedfs_mount:/mnt/seaweedfs - - /tmp/seaweedfs-mount-logs:/var/log/seaweedfs - networks: - - seaweedfs-rdma - environment: - - FILER_ADDR=seaweedfs-filer:8888 - - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 - - MOUNT_POINT=/mnt/seaweedfs - - RDMA_ENABLED=true - - RDMA_FALLBACK=true - - RDMA_MAX_CONCURRENT=64 - - RDMA_TIMEOUT_MS=5000 - - DEBUG=true - command: /usr/local/bin/mount-helper.sh - depends_on: - seaweedfs-filer: - condition: service_healthy - rdma-sidecar: - condition: service_healthy - healthcheck: - test: ["CMD", "mountpoint", "-q", "/mnt/seaweedfs"] - interval: 15s - timeout: 10s - retries: 3 - start_period: 45s - - # Integration Test Runner - integration-test: - build: - context: . - dockerfile: Dockerfile.integration-test - container_name: integration-test - volumes: - - seaweedfs_mount:/mnt/seaweedfs - - ./test-results:/test-results - networks: - - seaweedfs-rdma - environment: - - MOUNT_POINT=/mnt/seaweedfs - - FILER_ADDR=seaweedfs-filer:8888 - - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 - - TEST_RESULTS_DIR=/test-results - depends_on: - seaweedfs-mount: - condition: service_healthy - command: > - sh -c " - echo 'Starting RDMA Mount Integration Tests...' && - sleep 10 && - /usr/local/bin/run-integration-tests.sh - " - profiles: - - test - - # Performance Test Runner - performance-test: - build: - context: . - dockerfile: Dockerfile.performance-test - container_name: performance-test - volumes: - - seaweedfs_mount:/mnt/seaweedfs - - ./performance-results:/performance-results - networks: - - seaweedfs-rdma - environment: - - MOUNT_POINT=/mnt/seaweedfs - - RDMA_SIDECAR_ADDR=rdma-sidecar:8081 - - PERFORMANCE_RESULTS_DIR=/performance-results - depends_on: - seaweedfs-mount: - condition: service_healthy - command: > - sh -c " - echo 'Starting RDMA Mount Performance Tests...' && - sleep 10 && - /usr/local/bin/run-performance-tests.sh - " - profiles: - - performance - -volumes: - seaweedfs_master_data: - driver: local - seaweedfs_volume_data: - driver: local - seaweedfs_mount: - driver: local - driver_opts: - type: tmpfs - device: tmpfs - o: size=1g - rdma_socket: - driver: local - -networks: - seaweedfs-rdma: - driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 diff --git a/seaweedfs-rdma-sidecar/docker-compose.rdma-sim.yml b/seaweedfs-rdma-sidecar/docker-compose.rdma-sim.yml deleted file mode 100644 index fc8de08dc..000000000 --- a/seaweedfs-rdma-sidecar/docker-compose.rdma-sim.yml +++ /dev/null @@ -1,209 +0,0 @@ -services: - # SeaweedFS Master Server - seaweedfs-master: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-master - command: master -ip=seaweedfs-master -port=9333 -mdir=/data - ports: - - "9333:9333" - volumes: - - master-data:/data - networks: - - seaweedfs-rdma - healthcheck: - test: ["CMD", "pgrep", "-f", "weed"] - interval: 15s - timeout: 10s - retries: 5 - start_period: 30s - - # SeaweedFS Volume Server - seaweedfs-volume: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-volume - command: volume -master=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data - ports: - - "8080:8080" - volumes: - - volume-data:/data - depends_on: - seaweedfs-master: - condition: service_healthy - networks: - - seaweedfs-rdma - healthcheck: - test: ["CMD", "pgrep", "-f", "weed"] - interval: 15s - timeout: 10s - retries: 5 - start_period: 30s - - # RDMA Simulation Environment - rdma-simulation: - build: - context: . - dockerfile: docker/Dockerfile.rdma-simulation - container_name: rdma-simulation - privileged: true # Required for RDMA kernel module loading - environment: - - RDMA_DEVICE=rxe0 - - UCX_TLS=rc_verbs,ud_verbs,tcp - - UCX_LOG_LEVEL=info - volumes: - - /lib/modules:/lib/modules:ro # Host kernel modules - - /sys:/sys # Required for sysfs access - - rdma-simulation-data:/opt/rdma-sim/data - networks: - - seaweedfs-rdma - ports: - - "18515:18515" # RDMA application port - - "4791:4791" # RDMA CM port - - "4792:4792" # Additional RDMA port - command: | - bash -c " - echo '๐Ÿš€ Setting up RDMA simulation environment...' - sudo /opt/rdma-sim/setup-soft-roce.sh || echo 'RDMA setup failed, continuing...' - echo '๐Ÿ“‹ RDMA environment status:' - /opt/rdma-sim/test-rdma.sh || true - echo '๐Ÿ”ง UCX information:' - /opt/rdma-sim/ucx-info.sh || true - echo 'โœ… RDMA simulation ready - keeping container alive...' - tail -f /dev/null - " - healthcheck: - test: ["CMD", "test", "-f", "/opt/rdma-sim/setup-soft-roce.sh"] - interval: 30s - timeout: 10s - retries: 3 - start_period: 30s - - # Rust RDMA Engine (with RDMA simulation support) - rdma-engine: - build: - context: . - dockerfile: Dockerfile.rdma-engine - container_name: rdma-engine - environment: - - RUST_LOG=debug - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - # UCX configuration for real RDMA - - UCX_TLS=rc_verbs,ud_verbs,tcp,shm - - UCX_NET_DEVICES=all - - UCX_LOG_LEVEL=info - - UCX_RNDV_SCHEME=put_zcopy - - UCX_RNDV_THRESH=8192 - volumes: - - rdma-socket:/tmp - # Share network namespace with RDMA simulation for device access - network_mode: "container:rdma-simulation" - depends_on: - rdma-simulation: - condition: service_healthy - command: ["./rdma-engine-server", "--debug", "--ipc-socket", "/tmp/rdma-engine.sock"] - healthcheck: - test: ["CMD", "test", "-S", "/tmp/rdma-engine.sock"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - - # Go RDMA Sidecar / Demo Server - rdma-sidecar: - build: - context: . - dockerfile: Dockerfile.sidecar - container_name: rdma-sidecar - ports: - - "8081:8081" - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - VOLUME_SERVER_URL=http://seaweedfs-volume:8080 - - DEBUG=true - volumes: - - rdma-socket:/tmp - depends_on: - rdma-engine: - condition: service_healthy - seaweedfs-volume: - condition: service_healthy - networks: - - seaweedfs-rdma - command: [ - "./demo-server", - "--port", "8081", - "--rdma-socket", "/tmp/rdma-engine.sock", - "--volume-server", "http://seaweedfs-volume:8080", - "--enable-rdma", - "--debug" - ] - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 20s - - # Test Client for Integration Testing - test-client: - build: - context: . - dockerfile: Dockerfile.test-client - container_name: test-client - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - SIDECAR_URL=http://rdma-sidecar:8081 - - SEAWEEDFS_MASTER=http://seaweedfs-master:9333 - - SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080 - volumes: - - rdma-socket:/tmp - depends_on: - rdma-sidecar: - condition: service_healthy - networks: - - seaweedfs-rdma - profiles: - - testing - command: ["tail", "-f", "/dev/null"] # Keep container running for manual testing - - # Integration Test Runner with RDMA - integration-tests-rdma: - build: - context: . - dockerfile: Dockerfile.test-client - container_name: integration-tests-rdma - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - SIDECAR_URL=http://rdma-sidecar:8081 - - SEAWEEDFS_MASTER=http://seaweedfs-master:9333 - - SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080 - - RDMA_SIMULATION=true - volumes: - - rdma-socket:/tmp - - ./tests:/tests - depends_on: - rdma-sidecar: - condition: service_healthy - rdma-simulation: - condition: service_healthy - networks: - - seaweedfs-rdma - profiles: - - testing - command: ["/tests/run-integration-tests.sh"] - -volumes: - master-data: - driver: local - volume-data: - driver: local - rdma-socket: - driver: local - rdma-simulation-data: - driver: local - -networks: - seaweedfs-rdma: - driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 diff --git a/seaweedfs-rdma-sidecar/docker-compose.yml b/seaweedfs-rdma-sidecar/docker-compose.yml deleted file mode 100644 index b046ef8a0..000000000 --- a/seaweedfs-rdma-sidecar/docker-compose.yml +++ /dev/null @@ -1,157 +0,0 @@ -services: - # SeaweedFS Master Server - seaweedfs-master: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-master - command: master -ip=seaweedfs-master -port=9333 -mdir=/data - ports: - - "9333:9333" - volumes: - - master-data:/data - networks: - - seaweedfs-rdma - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9333/cluster/status"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 10s - - # SeaweedFS Volume Server - seaweedfs-volume: - image: chrislusf/seaweedfs:latest - container_name: seaweedfs-volume - command: volume -master=seaweedfs-master:9333 -ip=seaweedfs-volume -port=8080 -dir=/data - ports: - - "8080:8080" - volumes: - - volume-data:/data - depends_on: - seaweedfs-master: - condition: service_healthy - networks: - - seaweedfs-rdma - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8080/status"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - - # Rust RDMA Engine - rdma-engine: - build: - context: . - dockerfile: Dockerfile.rdma-engine.simple - container_name: rdma-engine - environment: - - RUST_LOG=debug - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - volumes: - - rdma-socket:/tmp - # Note: hugepages mount commented out to avoid host system requirements - # - /dev/hugepages:/dev/hugepages - # Privileged mode for RDMA access (in production, use specific capabilities) - privileged: true - networks: - - seaweedfs-rdma - command: ["./rdma-engine-server", "--debug", "--ipc-socket", "/tmp/rdma-engine.sock"] - healthcheck: - test: ["CMD", "test", "-S", "/tmp/rdma-engine.sock"] - interval: 5s - timeout: 3s - retries: 5 - start_period: 10s - - # Go RDMA Sidecar / Demo Server - rdma-sidecar: - build: - context: . - dockerfile: Dockerfile.sidecar - container_name: rdma-sidecar - ports: - - "8081:8081" - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - VOLUME_SERVER_URL=http://seaweedfs-volume:8080 - - DEBUG=true - volumes: - - rdma-socket:/tmp - depends_on: - rdma-engine: - condition: service_healthy - seaweedfs-volume: - condition: service_healthy - networks: - - seaweedfs-rdma - command: [ - "./demo-server", - "--port", "8081", - "--rdma-socket", "/tmp/rdma-engine.sock", - "--volume-server", "http://seaweedfs-volume:8080", - "--enable-rdma", - "--debug" - ] - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:8081/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 15s - - # Test Client for Integration Testing - test-client: - build: - context: . - dockerfile: Dockerfile.test-client - container_name: test-client - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - SIDECAR_URL=http://rdma-sidecar:8081 - - SEAWEEDFS_MASTER=http://seaweedfs-master:9333 - - SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080 - volumes: - - rdma-socket:/tmp - depends_on: - rdma-sidecar: - condition: service_healthy - networks: - - seaweedfs-rdma - profiles: - - testing - command: ["tail", "-f", "/dev/null"] # Keep container running for manual testing - - # Integration Test Runner - integration-tests: - build: - context: . - dockerfile: Dockerfile.test-client - container_name: integration-tests - environment: - - RDMA_SOCKET_PATH=/tmp/rdma-engine.sock - - SIDECAR_URL=http://rdma-sidecar:8081 - - SEAWEEDFS_MASTER=http://seaweedfs-master:9333 - - SEAWEEDFS_VOLUME=http://seaweedfs-volume:8080 - volumes: - - rdma-socket:/tmp - - ./tests:/tests - depends_on: - rdma-sidecar: - condition: service_healthy - networks: - - seaweedfs-rdma - profiles: - - testing - command: ["/tests/run-integration-tests.sh"] - -volumes: - master-data: - driver: local - volume-data: - driver: local - rdma-socket: - driver: local - -networks: - seaweedfs-rdma: - driver: bridge diff --git a/seaweedfs-rdma-sidecar/docker/Dockerfile.rdma-simulation b/seaweedfs-rdma-sidecar/docker/Dockerfile.rdma-simulation deleted file mode 100644 index 9f2566623..000000000 --- a/seaweedfs-rdma-sidecar/docker/Dockerfile.rdma-simulation +++ /dev/null @@ -1,77 +0,0 @@ -# RDMA Simulation Container with Soft-RoCE (RXE) -# This container enables software RDMA over regular Ethernet - -FROM ubuntu:22.04 - -# Install RDMA and networking tools -RUN apt-get update && apt-get install -y \ - # System utilities - sudo \ - # RDMA core libraries - libibverbs1 \ - libibverbs-dev \ - librdmacm1 \ - librdmacm-dev \ - rdma-core \ - ibverbs-utils \ - infiniband-diags \ - # Network tools - iproute2 \ - iputils-ping \ - net-tools \ - # Build tools - build-essential \ - pkg-config \ - cmake \ - # UCX dependencies - libnuma1 \ - libnuma-dev \ - # UCX library (pre-built) - try to install but don't fail if not available - # libucx0 \ - # libucx-dev \ - # Debugging tools - strace \ - gdb \ - valgrind \ - # Utilities - curl \ - wget \ - vim \ - htop \ - && rm -rf /var/lib/apt/lists/* - -# Try to install UCX tools (optional, may not be available in all repositories) -RUN apt-get update && \ - (apt-get install -y ucx-tools || echo "UCX tools not available in repository") && \ - rm -rf /var/lib/apt/lists/* - -# Create rdmauser for security (avoid conflict with system rdma group) -RUN useradd -m -s /bin/bash -G sudo,rdma rdmauser && \ - echo "rdmauser ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers - -# Create directories for RDMA setup -RUN mkdir -p /opt/rdma-sim /var/log/rdma - -# Copy RDMA simulation scripts -COPY docker/scripts/setup-soft-roce.sh /opt/rdma-sim/ -COPY docker/scripts/test-rdma.sh /opt/rdma-sim/ -COPY docker/scripts/ucx-info.sh /opt/rdma-sim/ - -# Make scripts executable -RUN chmod +x /opt/rdma-sim/*.sh - -# Set working directory -WORKDIR /opt/rdma-sim - -# Switch to rdmauser -USER rdmauser - -# Default command -CMD ["/bin/bash"] - -# Health check for RDMA devices -HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ - CMD /opt/rdma-sim/test-rdma.sh || exit 1 - -# Expose common RDMA ports -EXPOSE 18515 4791 4792 diff --git a/seaweedfs-rdma-sidecar/docker/scripts/setup-soft-roce.sh b/seaweedfs-rdma-sidecar/docker/scripts/setup-soft-roce.sh deleted file mode 100755 index 55c8f3b80..000000000 --- a/seaweedfs-rdma-sidecar/docker/scripts/setup-soft-roce.sh +++ /dev/null @@ -1,183 +0,0 @@ -#!/bin/bash - -# Setup Soft-RoCE (RXE) for RDMA simulation -# This script enables RDMA over Ethernet using the RXE kernel module - -set -e - -echo "๐Ÿ”ง Setting up Soft-RoCE (RXE) RDMA simulation..." - -# Function to check if running with required privileges -check_privileges() { - if [ "$EUID" -ne 0 ]; then - echo "โŒ This script requires root privileges" - echo "Run with: sudo $0 or inside a privileged container" - exit 1 - fi -} - -# Function to load RXE kernel module -load_rxe_module() { - echo "๐Ÿ“ฆ Loading RXE kernel module..." - - # Try to load the rdma_rxe module - if modprobe rdma_rxe 2>/dev/null; then - echo "โœ… rdma_rxe module loaded successfully" - else - echo "โš ๏ธ Failed to load rdma_rxe module, trying alternative approach..." - - # Alternative: Try loading rxe_net (older kernels) - if modprobe rxe_net 2>/dev/null; then - echo "โœ… rxe_net module loaded successfully" - else - echo "โŒ Failed to load RXE modules. Possible causes:" - echo " - Kernel doesn't support RXE (needs CONFIG_RDMA_RXE=m)" - echo " - Running in unprivileged container" - echo " - Missing kernel modules" - echo "" - echo "๐Ÿ”ง Workaround: Run container with --privileged flag" - exit 1 - fi - fi - - # Verify module is loaded - if lsmod | grep -q "rdma_rxe\|rxe_net"; then - echo "โœ… RXE module verification successful" - else - echo "โŒ RXE module verification failed" - exit 1 - fi -} - -# Function to setup virtual RDMA device -setup_rxe_device() { - echo "๐ŸŒ Setting up RXE device over Ethernet interface..." - - # Find available network interface (prefer eth0, fallback to others) - local interface="" - for iface in eth0 enp0s3 enp0s8 lo; do - if ip link show "$iface" >/dev/null 2>&1; then - interface="$iface" - break - fi - done - - if [ -z "$interface" ]; then - echo "โŒ No suitable network interface found" - echo "Available interfaces:" - ip link show | grep "^[0-9]" | cut -d':' -f2 | tr -d ' ' - exit 1 - fi - - echo "๐Ÿ“ก Using network interface: $interface" - - # Create RXE device - echo "๐Ÿ”จ Creating RXE device on $interface..." - - # Try modern rxe_cfg approach first - if command -v rxe_cfg >/dev/null 2>&1; then - rxe_cfg add "$interface" || { - echo "โš ๏ธ rxe_cfg failed, trying manual approach..." - setup_rxe_manual "$interface" - } - else - echo "โš ๏ธ rxe_cfg not available, using manual setup..." - setup_rxe_manual "$interface" - fi -} - -# Function to manually setup RXE device -setup_rxe_manual() { - local interface="$1" - - # Use sysfs interface to create RXE device - if [ -d /sys/module/rdma_rxe ]; then - echo "$interface" > /sys/module/rdma_rxe/parameters/add 2>/dev/null || { - echo "โŒ Failed to add RXE device via sysfs" - exit 1 - } - else - echo "โŒ RXE sysfs interface not found" - exit 1 - fi -} - -# Function to verify RDMA devices -verify_rdma_devices() { - echo "๐Ÿ” Verifying RDMA devices..." - - # Check for RDMA devices - if [ -d /sys/class/infiniband ]; then - local devices=$(ls /sys/class/infiniband/ 2>/dev/null | wc -l) - if [ "$devices" -gt 0 ]; then - echo "โœ… Found $devices RDMA device(s):" - ls /sys/class/infiniband/ - - # Show device details - for device in /sys/class/infiniband/*; do - if [ -d "$device" ]; then - local dev_name=$(basename "$device") - echo " ๐Ÿ“‹ Device: $dev_name" - - # Try to get device info - if command -v ibv_devinfo >/dev/null 2>&1; then - ibv_devinfo -d "$dev_name" | head -10 - fi - fi - done - else - echo "โŒ No RDMA devices found in /sys/class/infiniband/" - exit 1 - fi - else - echo "โŒ /sys/class/infiniband directory not found" - exit 1 - fi -} - -# Function to test basic RDMA functionality -test_basic_rdma() { - echo "๐Ÿงช Testing basic RDMA functionality..." - - # Test libibverbs - if command -v ibv_devinfo >/dev/null 2>&1; then - echo "๐Ÿ“‹ RDMA device information:" - ibv_devinfo | head -20 - else - echo "โš ๏ธ ibv_devinfo not available" - fi - - # Test UCX if available - if command -v ucx_info >/dev/null 2>&1; then - echo "๐Ÿ“‹ UCX information:" - ucx_info -d | head -10 - else - echo "โš ๏ธ UCX tools not available" - fi -} - -# Main execution -main() { - echo "๐Ÿš€ Starting Soft-RoCE RDMA simulation setup..." - echo "======================================" - - check_privileges - load_rxe_module - setup_rxe_device - verify_rdma_devices - test_basic_rdma - - echo "" - echo "๐ŸŽ‰ Soft-RoCE setup completed successfully!" - echo "======================================" - echo "โœ… RDMA simulation is ready for testing" - echo "๐Ÿ“ก You can now run RDMA applications" - echo "" - echo "Next steps:" - echo " - Test with: /opt/rdma-sim/test-rdma.sh" - echo " - Check UCX: /opt/rdma-sim/ucx-info.sh" - echo " - Run your RDMA applications" -} - -# Execute main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/docker/scripts/test-rdma.sh b/seaweedfs-rdma-sidecar/docker/scripts/test-rdma.sh deleted file mode 100755 index 91e60ca7f..000000000 --- a/seaweedfs-rdma-sidecar/docker/scripts/test-rdma.sh +++ /dev/null @@ -1,253 +0,0 @@ -#!/bin/bash - -# Test RDMA functionality in simulation environment -# This script validates that RDMA devices and libraries are working - -set -e - -echo "๐Ÿงช Testing RDMA simulation environment..." - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Function to print colored output -print_status() { - local status="$1" - local message="$2" - - case "$status" in - "success") - echo -e "${GREEN}โœ… $message${NC}" - ;; - "warning") - echo -e "${YELLOW}โš ๏ธ $message${NC}" - ;; - "error") - echo -e "${RED}โŒ $message${NC}" - ;; - "info") - echo -e "${BLUE}๐Ÿ“‹ $message${NC}" - ;; - esac -} - -# Function to test RDMA devices -test_rdma_devices() { - print_status "info" "Testing RDMA devices..." - - # Check for InfiniBand/RDMA devices - if [ -d /sys/class/infiniband ]; then - local device_count=$(ls /sys/class/infiniband/ 2>/dev/null | wc -l) - if [ "$device_count" -gt 0 ]; then - print_status "success" "Found $device_count RDMA device(s)" - - # List devices - for device in /sys/class/infiniband/*; do - if [ -d "$device" ]; then - local dev_name=$(basename "$device") - print_status "info" "Device: $dev_name" - fi - done - return 0 - else - print_status "error" "No RDMA devices found" - return 1 - fi - else - print_status "error" "/sys/class/infiniband directory not found" - return 1 - fi -} - -# Function to test libibverbs -test_libibverbs() { - print_status "info" "Testing libibverbs..." - - if command -v ibv_devinfo >/dev/null 2>&1; then - # Get device info - local device_info=$(ibv_devinfo 2>/dev/null) - if [ -n "$device_info" ]; then - print_status "success" "libibverbs working - devices detected" - - # Show basic info - echo "$device_info" | head -5 - - # Test device capabilities - if echo "$device_info" | grep -q "transport.*InfiniBand\|transport.*Ethernet"; then - print_status "success" "RDMA transport layer detected" - else - print_status "warning" "Transport layer information unclear" - fi - - return 0 - else - print_status "error" "ibv_devinfo found no devices" - return 1 - fi - else - print_status "error" "ibv_devinfo command not found" - return 1 - fi -} - -# Function to test UCX -test_ucx() { - print_status "info" "Testing UCX..." - - if command -v ucx_info >/dev/null 2>&1; then - # Test UCX device detection - local ucx_output=$(ucx_info -d 2>/dev/null) - if [ -n "$ucx_output" ]; then - print_status "success" "UCX detecting devices" - - # Show UCX device info - echo "$ucx_output" | head -10 - - # Check for RDMA transports - if echo "$ucx_output" | grep -q "rc\|ud\|dc"; then - print_status "success" "UCX RDMA transports available" - else - print_status "warning" "UCX RDMA transports not detected" - fi - - return 0 - else - print_status "warning" "UCX not detecting devices" - return 1 - fi - else - print_status "warning" "UCX tools not available" - return 1 - fi -} - -# Function to test RDMA CM (Connection Manager) -test_rdma_cm() { - print_status "info" "Testing RDMA Connection Manager..." - - # Check for RDMA CM device - if [ -e /dev/infiniband/rdma_cm ]; then - print_status "success" "RDMA CM device found" - return 0 - else - print_status "warning" "RDMA CM device not found" - return 1 - fi -} - -# Function to test basic RDMA operations -test_rdma_operations() { - print_status "info" "Testing basic RDMA operations..." - - # Try to run a simple RDMA test if tools are available - if command -v ibv_rc_pingpong >/dev/null 2>&1; then - # This would need a client/server setup, so just check if binary exists - print_status "success" "RDMA test tools available (ibv_rc_pingpong)" - else - print_status "warning" "RDMA test tools not available" - fi - - # Check for other useful RDMA utilities - local tools_found=0 - for tool in ibv_asyncwatch ibv_read_lat ibv_write_lat; do - if command -v "$tool" >/dev/null 2>&1; then - tools_found=$((tools_found + 1)) - fi - done - - if [ "$tools_found" -gt 0 ]; then - print_status "success" "Found $tools_found additional RDMA test tools" - else - print_status "warning" "No additional RDMA test tools found" - fi -} - -# Function to generate test summary -generate_summary() { - echo "" - print_status "info" "RDMA Simulation Test Summary" - echo "======================================" - - # Re-run key tests for summary - local devices_ok=0 - local libibverbs_ok=0 - local ucx_ok=0 - - if [ -d /sys/class/infiniband ] && [ "$(ls /sys/class/infiniband/ 2>/dev/null | wc -l)" -gt 0 ]; then - devices_ok=1 - fi - - if command -v ibv_devinfo >/dev/null 2>&1 && ibv_devinfo >/dev/null 2>&1; then - libibverbs_ok=1 - fi - - if command -v ucx_info >/dev/null 2>&1 && ucx_info -d >/dev/null 2>&1; then - ucx_ok=1 - fi - - echo "๐Ÿ“Š Test Results:" - [ "$devices_ok" -eq 1 ] && print_status "success" "RDMA Devices: PASS" || print_status "error" "RDMA Devices: FAIL" - [ "$libibverbs_ok" -eq 1 ] && print_status "success" "libibverbs: PASS" || print_status "error" "libibverbs: FAIL" - [ "$ucx_ok" -eq 1 ] && print_status "success" "UCX: PASS" || print_status "warning" "UCX: FAIL/WARNING" - - echo "" - if [ "$devices_ok" -eq 1 ] && [ "$libibverbs_ok" -eq 1 ]; then - print_status "success" "RDMA simulation environment is ready! ๐ŸŽ‰" - echo "" - print_status "info" "You can now:" - echo " - Run RDMA applications" - echo " - Test SeaweedFS RDMA engine with real RDMA" - echo " - Use UCX for high-performance transfers" - return 0 - else - print_status "error" "RDMA simulation setup needs attention" - echo "" - print_status "info" "Troubleshooting:" - echo " - Run setup script: sudo /opt/rdma-sim/setup-soft-roce.sh" - echo " - Check container privileges (--privileged flag)" - echo " - Verify kernel RDMA support" - return 1 - fi -} - -# Main test execution -main() { - echo "๐Ÿš€ RDMA Simulation Test Suite" - echo "======================================" - - # Run tests - test_rdma_devices || true - echo "" - - test_libibverbs || true - echo "" - - test_ucx || true - echo "" - - test_rdma_cm || true - echo "" - - test_rdma_operations || true - echo "" - - # Generate summary - generate_summary -} - -# Health check mode (for Docker healthcheck) -if [ "$1" = "healthcheck" ]; then - # Quick health check - just verify devices exist - if [ -d /sys/class/infiniband ] && [ "$(ls /sys/class/infiniband/ 2>/dev/null | wc -l)" -gt 0 ]; then - exit 0 - else - exit 1 - fi -fi - -# Execute main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/docker/scripts/ucx-info.sh b/seaweedfs-rdma-sidecar/docker/scripts/ucx-info.sh deleted file mode 100755 index 9bf287c6e..000000000 --- a/seaweedfs-rdma-sidecar/docker/scripts/ucx-info.sh +++ /dev/null @@ -1,269 +0,0 @@ -#!/bin/bash - -# UCX Information and Testing Script -# Provides detailed information about UCX configuration and capabilities - -set -e - -echo "๐Ÿ“‹ UCX (Unified Communication X) Information" -echo "=============================================" - -# Colors for output -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -print_section() { - echo -e "\n${BLUE}๐Ÿ“Œ $1${NC}" - echo "----------------------------------------" -} - -print_info() { - echo -e "${GREEN}โ„น๏ธ $1${NC}" -} - -print_warning() { - echo -e "${YELLOW}โš ๏ธ $1${NC}" -} - -# Function to check UCX installation -check_ucx_installation() { - print_section "UCX Installation Status" - - if command -v ucx_info >/dev/null 2>&1; then - print_info "UCX tools are installed" - - # Get UCX version - if ucx_info -v >/dev/null 2>&1; then - local version=$(ucx_info -v 2>/dev/null | head -1) - print_info "Version: $version" - fi - else - print_warning "UCX tools not found" - echo "Install with: apt-get install ucx-tools libucx-dev" - return 1 - fi - - # Check UCX libraries - local libs_found=0 - for lib in libucp.so libucs.so libuct.so; do - if ldconfig -p | grep -q "$lib"; then - libs_found=$((libs_found + 1)) - fi - done - - if [ "$libs_found" -eq 3 ]; then - print_info "All UCX libraries found (ucp, ucs, uct)" - else - print_warning "Some UCX libraries may be missing ($libs_found/3 found)" - fi -} - -# Function to show UCX device information -show_ucx_devices() { - print_section "UCX Transport Devices" - - if command -v ucx_info >/dev/null 2>&1; then - echo "Available UCX transports and devices:" - ucx_info -d 2>/dev/null || { - print_warning "Failed to get UCX device information" - return 1 - } - else - print_warning "ucx_info command not available" - return 1 - fi -} - -# Function to show UCX configuration -show_ucx_config() { - print_section "UCX Configuration" - - if command -v ucx_info >/dev/null 2>&1; then - echo "UCX configuration parameters:" - ucx_info -c 2>/dev/null | head -20 || { - print_warning "Failed to get UCX configuration" - return 1 - } - - echo "" - print_info "Key UCX environment variables:" - echo " UCX_TLS - Transport layers to use" - echo " UCX_NET_DEVICES - Network devices to use" - echo " UCX_LOG_LEVEL - Logging level (error, warn, info, debug, trace)" - echo " UCX_MEMTYPE_CACHE - Memory type caching (y/n)" - else - print_warning "ucx_info command not available" - return 1 - fi -} - -# Function to test UCX capabilities -test_ucx_capabilities() { - print_section "UCX Capability Testing" - - if command -v ucx_info >/dev/null 2>&1; then - print_info "Testing UCX transport capabilities..." - - # Check for RDMA transports - local ucx_transports=$(ucx_info -d 2>/dev/null | grep -i "transport\|tl:" || true) - - if echo "$ucx_transports" | grep -q "rc\|dc\|ud"; then - print_info "โœ… RDMA transports detected (RC/DC/UD)" - else - print_warning "No RDMA transports detected" - fi - - if echo "$ucx_transports" | grep -q "tcp"; then - print_info "โœ… TCP transport available" - else - print_warning "TCP transport not detected" - fi - - if echo "$ucx_transports" | grep -q "shm\|posix"; then - print_info "โœ… Shared memory transport available" - else - print_warning "Shared memory transport not detected" - fi - - # Memory types - print_info "Testing memory type support..." - local memory_info=$(ucx_info -d 2>/dev/null | grep -i "memory\|md:" || true) - if [ -n "$memory_info" ]; then - echo "$memory_info" | head -5 - fi - - else - print_warning "Cannot test UCX capabilities - ucx_info not available" - return 1 - fi -} - -# Function to show recommended UCX settings for RDMA -show_rdma_settings() { - print_section "Recommended UCX Settings for RDMA" - - print_info "For optimal RDMA performance with SeaweedFS:" - echo "" - echo "Environment Variables:" - echo " export UCX_TLS=rc_verbs,ud_verbs,rc_mlx5_dv,dc_mlx5_dv" - echo " export UCX_NET_DEVICES=all" - echo " export UCX_LOG_LEVEL=info" - echo " export UCX_RNDV_SCHEME=put_zcopy" - echo " export UCX_RNDV_THRESH=8192" - echo "" - - print_info "For development/debugging:" - echo " export UCX_LOG_LEVEL=debug" - echo " export UCX_LOG_FILE=/tmp/ucx.log" - echo "" - - print_info "For Soft-RoCE (RXE) specifically:" - echo " export UCX_TLS=rc_verbs,ud_verbs" - echo " export UCX_IB_DEVICE_SPECS=rxe0:1" - echo "" -} - -# Function to test basic UCX functionality -test_ucx_basic() { - print_section "Basic UCX Functionality Test" - - if command -v ucx_hello_world >/dev/null 2>&1; then - print_info "UCX hello_world test available" - echo "You can test UCX with:" - echo " Server: UCX_TLS=tcp ucx_hello_world -l" - echo " Client: UCX_TLS=tcp ucx_hello_world " - else - print_warning "UCX hello_world test not available" - fi - - # Check for other UCX test utilities - local test_tools=0 - for tool in ucx_perftest ucp_hello_world; do - if command -v "$tool" >/dev/null 2>&1; then - test_tools=$((test_tools + 1)) - print_info "UCX test tool available: $tool" - fi - done - - if [ "$test_tools" -eq 0 ]; then - print_warning "No UCX test tools found" - echo "Consider installing: ucx-tools package" - fi -} - -# Function to generate UCX summary -generate_summary() { - print_section "UCX Status Summary" - - local ucx_ok=0 - local devices_ok=0 - local rdma_ok=0 - - # Check UCX availability - if command -v ucx_info >/dev/null 2>&1; then - ucx_ok=1 - fi - - # Check devices - if command -v ucx_info >/dev/null 2>&1 && ucx_info -d >/dev/null 2>&1; then - devices_ok=1 - - # Check for RDMA - if ucx_info -d 2>/dev/null | grep -q "rc\|dc\|ud"; then - rdma_ok=1 - fi - fi - - echo "๐Ÿ“Š UCX Status:" - [ "$ucx_ok" -eq 1 ] && print_info "โœ… UCX Installation: OK" || print_warning "โŒ UCX Installation: Missing" - [ "$devices_ok" -eq 1 ] && print_info "โœ… UCX Devices: Detected" || print_warning "โŒ UCX Devices: Not detected" - [ "$rdma_ok" -eq 1 ] && print_info "โœ… RDMA Support: Available" || print_warning "โš ๏ธ RDMA Support: Limited/Missing" - - echo "" - if [ "$ucx_ok" -eq 1 ] && [ "$devices_ok" -eq 1 ]; then - print_info "๐ŸŽ‰ UCX is ready for SeaweedFS RDMA integration!" - - if [ "$rdma_ok" -eq 1 ]; then - print_info "๐Ÿš€ Real RDMA acceleration is available" - else - print_warning "๐Ÿ’ก Only TCP/shared memory transports available" - fi - else - print_warning "๐Ÿ”ง UCX setup needs attention for optimal performance" - fi -} - -# Main execution -main() { - check_ucx_installation - echo "" - - show_ucx_devices - echo "" - - show_ucx_config - echo "" - - test_ucx_capabilities - echo "" - - show_rdma_settings - echo "" - - test_ucx_basic - echo "" - - generate_summary - - echo "" - print_info "For SeaweedFS RDMA engine integration:" - echo " 1. Use UCX with your Rust engine" - echo " 2. Configure appropriate transport layers" - echo " 3. Test with SeaweedFS RDMA sidecar" - echo " 4. Monitor performance and adjust settings" -} - -# Execute main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/go.mod b/seaweedfs-rdma-sidecar/go.mod deleted file mode 100644 index c4db7eccf..000000000 --- a/seaweedfs-rdma-sidecar/go.mod +++ /dev/null @@ -1,50 +0,0 @@ -module seaweedfs-rdma-sidecar - -go 1.26 - -require ( - github.com/seaweedfs/seaweedfs v0.0.0-00010101000000-000000000000 - github.com/sirupsen/logrus v1.9.4 - github.com/spf13/cobra v1.8.0 - github.com/vmihailenco/msgpack/v5 v5.4.1 -) - -require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cognusion/imaging v1.0.4 // indirect - github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/go-viper/mapstructure/v2 v2.5.0 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/prometheus/client_golang v1.24.1 // indirect - github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.70.1 // indirect - github.com/prometheus/procfs v0.21.1 // indirect - github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect - github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/seaweedfs/goexif v2.0.0+incompatible // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect - github.com/spf13/afero v1.15.0 // indirect - github.com/spf13/cast v1.10.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect - github.com/spf13/viper v1.21.0 // indirect - github.com/subosito/gotenv v1.6.0 // indirect - github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/image v0.44.0 // indirect - golang.org/x/net v0.58.0 // indirect - golang.org/x/sys v0.47.0 // indirect - golang.org/x/text v0.41.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect - google.golang.org/grpc v1.85.0-dev // indirect - google.golang.org/protobuf v1.36.12 // indirect -) - -// For local development, this replace directive is required to build the sidecar -// against the parent SeaweedFS module in this monorepo. -// -// To build this module, ensure the main SeaweedFS repository is checked out -// as a sibling directory to this `seaweedfs-rdma-sidecar` directory. -replace github.com/seaweedfs/seaweedfs => ../ diff --git a/seaweedfs-rdma-sidecar/go.sum b/seaweedfs-rdma-sidecar/go.sum deleted file mode 100644 index 3cbda81b6..000000000 --- a/seaweedfs-rdma-sidecar/go.sum +++ /dev/null @@ -1,102 +0,0 @@ -github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= -github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cognusion/imaging v1.0.4 h1:hrmWVa4S9fiLJJFKEHYYRNBUnHEKZD8/oxnC77xTUa0= -github.com/cognusion/imaging v1.0.4/go.mod h1:X1mPSSZpMyiva54chCO/1NROp+b6xG4uMcud8VLii+U= -github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= -github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= -github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= -github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= -github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= -github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= -github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= -github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= -github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= -github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= -github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= -github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= -github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= -github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/seaweedfs/goexif v2.0.0+incompatible h1:x8pckiT12QQhifwhDQpeISgDfsqmQ6VR4LFPQ64JRps= -github.com/seaweedfs/goexif v2.0.0+incompatible/go.mod h1:Oni780Z236sXpIQzk1XoJlTwqrJ02smEin9zQeff7Fk= -github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= -github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= -github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= -github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= -github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= -github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= -github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= -github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= -github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= -github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= -github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok= -github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= -github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= -go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/image v0.44.0 h1:+tDekMZED9+LrtB3G5xzRggpVh9CARjZqROla3R3R+I= -golang.org/x/image v0.44.0/go.mod h1:V8K3KE9KKKE+pLpQDOeN18w9oacNSvy1tDOirTu4xtY= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= -gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 h1:cYNAzI2sUwhmCcoj9TxvihSrqsxt6uIkj3rDRhSDmW4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= -google.golang.org/grpc v1.85.0-dev h1:HxkDyKIIZPpFnroC56tQv5gNuKTmVvi0t7TzOf5zt7g= -google.golang.org/grpc v1.85.0-dev/go.mod h1:ljCht0DrxQrXBDRTZp52Qxh3Ffk8CdYm2sj4O2QN2C0= -google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= -google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/seaweedfs-rdma-sidecar/pkg/ipc/client.go b/seaweedfs-rdma-sidecar/pkg/ipc/client.go deleted file mode 100644 index b2c1d2db1..000000000 --- a/seaweedfs-rdma-sidecar/pkg/ipc/client.go +++ /dev/null @@ -1,331 +0,0 @@ -package ipc - -import ( - "context" - "encoding/binary" - "fmt" - "net" - "sync" - "time" - - "github.com/sirupsen/logrus" - "github.com/vmihailenco/msgpack/v5" -) - -// Client provides IPC communication with the Rust RDMA engine -type Client struct { - socketPath string - conn net.Conn - mu sync.RWMutex - logger *logrus.Logger - connected bool -} - -// NewClient creates a new IPC client -func NewClient(socketPath string, logger *logrus.Logger) *Client { - if logger == nil { - logger = logrus.New() - logger.SetLevel(logrus.InfoLevel) - } - - return &Client{ - socketPath: socketPath, - logger: logger, - } -} - -// Connect establishes connection to the Rust RDMA engine -func (c *Client) Connect(ctx context.Context) error { - c.mu.Lock() - defer c.mu.Unlock() - - if c.connected { - return nil - } - - c.logger.WithField("socket", c.socketPath).Info("๐Ÿ”— Connecting to Rust RDMA engine") - - dialer := &net.Dialer{} - conn, err := dialer.DialContext(ctx, "unix", c.socketPath) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to connect to RDMA engine") - return fmt.Errorf("failed to connect to RDMA engine at %s: %w", c.socketPath, err) - } - - c.conn = conn - c.connected = true - c.logger.Info("โœ… Connected to Rust RDMA engine") - - return nil -} - -// Disconnect closes the connection -func (c *Client) Disconnect() { - c.mu.Lock() - defer c.mu.Unlock() - - if c.conn != nil { - c.conn.Close() - c.conn = nil - c.connected = false - c.logger.Info("๐Ÿ”Œ Disconnected from Rust RDMA engine") - } -} - -// IsConnected returns connection status -func (c *Client) IsConnected() bool { - c.mu.RLock() - defer c.mu.RUnlock() - return c.connected -} - -// SendMessage sends an IPC message and waits for response -func (c *Client) SendMessage(ctx context.Context, msg *IpcMessage) (*IpcMessage, error) { - c.mu.RLock() - conn := c.conn - connected := c.connected - c.mu.RUnlock() - - if !connected || conn == nil { - return nil, fmt.Errorf("not connected to RDMA engine") - } - - // Set write timeout - if deadline, ok := ctx.Deadline(); ok { - conn.SetWriteDeadline(deadline) - } else { - conn.SetWriteDeadline(time.Now().Add(30 * time.Second)) - } - - c.logger.WithField("type", msg.Type).Debug("๐Ÿ“ค Sending message to Rust engine") - - // Serialize message with MessagePack - data, err := msgpack.Marshal(msg) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to marshal message") - return nil, fmt.Errorf("failed to marshal message: %w", err) - } - - // Send message length (4 bytes) + message data - lengthBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(lengthBytes, uint32(len(data))) - - if _, err := conn.Write(lengthBytes); err != nil { - c.logger.WithError(err).Error("โŒ Failed to send message length") - return nil, fmt.Errorf("failed to send message length: %w", err) - } - - if _, err := conn.Write(data); err != nil { - c.logger.WithError(err).Error("โŒ Failed to send message data") - return nil, fmt.Errorf("failed to send message data: %w", err) - } - - c.logger.WithFields(logrus.Fields{ - "type": msg.Type, - "size": len(data), - }).Debug("๐Ÿ“ค Message sent successfully") - - // Read response - return c.readResponse(ctx, conn) -} - -// readResponse reads and deserializes the response message -func (c *Client) readResponse(ctx context.Context, conn net.Conn) (*IpcMessage, error) { - // Set read timeout - if deadline, ok := ctx.Deadline(); ok { - conn.SetReadDeadline(deadline) - } else { - conn.SetReadDeadline(time.Now().Add(30 * time.Second)) - } - - // Read message length (4 bytes) - lengthBytes := make([]byte, 4) - if _, err := conn.Read(lengthBytes); err != nil { - c.logger.WithError(err).Error("โŒ Failed to read response length") - return nil, fmt.Errorf("failed to read response length: %w", err) - } - - length := binary.LittleEndian.Uint32(lengthBytes) - if length > 64*1024*1024 { // 64MB sanity check - c.logger.WithField("length", length).Error("โŒ Response message too large") - return nil, fmt.Errorf("response message too large: %d bytes", length) - } - - // Read message data - data := make([]byte, length) - if _, err := conn.Read(data); err != nil { - c.logger.WithError(err).Error("โŒ Failed to read response data") - return nil, fmt.Errorf("failed to read response data: %w", err) - } - - c.logger.WithField("size", length).Debug("๐Ÿ“ฅ Response received") - - // Deserialize with MessagePack - var response IpcMessage - if err := msgpack.Unmarshal(data, &response); err != nil { - c.logger.WithError(err).Error("โŒ Failed to unmarshal response") - return nil, fmt.Errorf("failed to unmarshal response: %w", err) - } - - c.logger.WithField("type", response.Type).Debug("๐Ÿ“ฅ Response deserialized successfully") - - return &response, nil -} - -// High-level convenience methods - -// Ping sends a ping message to test connectivity -func (c *Client) Ping(ctx context.Context, clientID *string) (*PongResponse, error) { - msg := NewPingMessage(clientID) - - response, err := c.SendMessage(ctx, msg) - if err != nil { - return nil, err - } - - if response.Type == MsgError { - errorData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal engine error data: %w", err) - } - var errorResp ErrorResponse - if err := msgpack.Unmarshal(errorData, &errorResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal engine error response: %w", err) - } - return nil, fmt.Errorf("engine error: %s - %s", errorResp.Code, errorResp.Message) - } - - if response.Type != MsgPong { - return nil, fmt.Errorf("unexpected response type: %s", response.Type) - } - - // Convert response data to PongResponse - pongData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal pong data: %w", err) - } - - var pong PongResponse - if err := msgpack.Unmarshal(pongData, &pong); err != nil { - return nil, fmt.Errorf("failed to unmarshal pong response: %w", err) - } - - return &pong, nil -} - -// GetCapabilities requests engine capabilities -func (c *Client) GetCapabilities(ctx context.Context, clientID *string) (*GetCapabilitiesResponse, error) { - msg := NewGetCapabilitiesMessage(clientID) - - response, err := c.SendMessage(ctx, msg) - if err != nil { - return nil, err - } - - if response.Type == MsgError { - errorData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal engine error data: %w", err) - } - var errorResp ErrorResponse - if err := msgpack.Unmarshal(errorData, &errorResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal engine error response: %w", err) - } - return nil, fmt.Errorf("engine error: %s - %s", errorResp.Code, errorResp.Message) - } - - if response.Type != MsgGetCapabilitiesResponse { - return nil, fmt.Errorf("unexpected response type: %s", response.Type) - } - - // Convert response data to GetCapabilitiesResponse - capsData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal capabilities data: %w", err) - } - - var caps GetCapabilitiesResponse - if err := msgpack.Unmarshal(capsData, &caps); err != nil { - return nil, fmt.Errorf("failed to unmarshal capabilities response: %w", err) - } - - return &caps, nil -} - -// StartRead initiates an RDMA read operation -func (c *Client) StartRead(ctx context.Context, req *StartReadRequest) (*StartReadResponse, error) { - msg := NewStartReadMessage(req) - - response, err := c.SendMessage(ctx, msg) - if err != nil { - return nil, err - } - - if response.Type == MsgError { - errorData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal engine error data: %w", err) - } - var errorResp ErrorResponse - if err := msgpack.Unmarshal(errorData, &errorResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal engine error response: %w", err) - } - return nil, fmt.Errorf("engine error: %s - %s", errorResp.Code, errorResp.Message) - } - - if response.Type != MsgStartReadResponse { - return nil, fmt.Errorf("unexpected response type: %s", response.Type) - } - - // Convert response data to StartReadResponse - startData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal start read data: %w", err) - } - - var startResp StartReadResponse - if err := msgpack.Unmarshal(startData, &startResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal start read response: %w", err) - } - - return &startResp, nil -} - -// CompleteRead completes an RDMA read operation -func (c *Client) CompleteRead(ctx context.Context, sessionID string, success bool, bytesTransferred uint64, clientCrc *uint32) (*CompleteReadResponse, error) { - msg := NewCompleteReadMessage(sessionID, success, bytesTransferred, clientCrc, nil) - - response, err := c.SendMessage(ctx, msg) - if err != nil { - return nil, err - } - - if response.Type == MsgError { - errorData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal engine error data: %w", err) - } - var errorResp ErrorResponse - if err := msgpack.Unmarshal(errorData, &errorResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal engine error response: %w", err) - } - return nil, fmt.Errorf("engine error: %s - %s", errorResp.Code, errorResp.Message) - } - - if response.Type != MsgCompleteReadResponse { - return nil, fmt.Errorf("unexpected response type: %s", response.Type) - } - - // Convert response data to CompleteReadResponse - completeData, err := msgpack.Marshal(response.Data) - if err != nil { - return nil, fmt.Errorf("failed to marshal complete read data: %w", err) - } - - var completeResp CompleteReadResponse - if err := msgpack.Unmarshal(completeData, &completeResp); err != nil { - return nil, fmt.Errorf("failed to unmarshal complete read response: %w", err) - } - - return &completeResp, nil -} diff --git a/seaweedfs-rdma-sidecar/pkg/ipc/messages.go b/seaweedfs-rdma-sidecar/pkg/ipc/messages.go deleted file mode 100644 index 4293ac396..000000000 --- a/seaweedfs-rdma-sidecar/pkg/ipc/messages.go +++ /dev/null @@ -1,160 +0,0 @@ -// Package ipc provides communication between Go sidecar and Rust RDMA engine -package ipc - -import "time" - -// IpcMessage represents the tagged union of all IPC messages -// This matches the Rust enum: #[serde(tag = "type", content = "data")] -type IpcMessage struct { - Type string `msgpack:"type"` - Data interface{} `msgpack:"data"` -} - -// Request message types -const ( - MsgStartRead = "StartRead" - MsgCompleteRead = "CompleteRead" - MsgGetCapabilities = "GetCapabilities" - MsgPing = "Ping" -) - -// Response message types -const ( - MsgStartReadResponse = "StartReadResponse" - MsgCompleteReadResponse = "CompleteReadResponse" - MsgGetCapabilitiesResponse = "GetCapabilitiesResponse" - MsgPong = "Pong" - MsgError = "Error" -) - -// StartReadRequest corresponds to Rust StartReadRequest -type StartReadRequest struct { - VolumeID uint32 `msgpack:"volume_id"` - NeedleID uint64 `msgpack:"needle_id"` - Cookie uint32 `msgpack:"cookie"` - Offset uint64 `msgpack:"offset"` - Size uint64 `msgpack:"size"` - RemoteAddr uint64 `msgpack:"remote_addr"` - RemoteKey uint32 `msgpack:"remote_key"` - TimeoutSecs uint64 `msgpack:"timeout_secs"` - AuthToken *string `msgpack:"auth_token,omitempty"` -} - -// StartReadResponse corresponds to Rust StartReadResponse -type StartReadResponse struct { - SessionID string `msgpack:"session_id"` - LocalAddr uint64 `msgpack:"local_addr"` - LocalKey uint32 `msgpack:"local_key"` - TransferSize uint64 `msgpack:"transfer_size"` - ExpectedCrc uint32 `msgpack:"expected_crc"` - ExpiresAtNs uint64 `msgpack:"expires_at_ns"` -} - -// CompleteReadRequest corresponds to Rust CompleteReadRequest -type CompleteReadRequest struct { - SessionID string `msgpack:"session_id"` - Success bool `msgpack:"success"` - BytesTransferred uint64 `msgpack:"bytes_transferred"` - ClientCrc *uint32 `msgpack:"client_crc,omitempty"` - ErrorMessage *string `msgpack:"error_message,omitempty"` -} - -// CompleteReadResponse corresponds to Rust CompleteReadResponse -type CompleteReadResponse struct { - Success bool `msgpack:"success"` - ServerCrc *uint32 `msgpack:"server_crc,omitempty"` - Message *string `msgpack:"message,omitempty"` -} - -// GetCapabilitiesRequest corresponds to Rust GetCapabilitiesRequest -type GetCapabilitiesRequest struct { - ClientID *string `msgpack:"client_id,omitempty"` -} - -// GetCapabilitiesResponse corresponds to Rust GetCapabilitiesResponse -type GetCapabilitiesResponse struct { - DeviceName string `msgpack:"device_name"` - VendorId uint32 `msgpack:"vendor_id"` - MaxTransferSize uint64 `msgpack:"max_transfer_size"` - MaxSessions usize `msgpack:"max_sessions"` - ActiveSessions usize `msgpack:"active_sessions"` - PortGid string `msgpack:"port_gid"` - PortLid uint16 `msgpack:"port_lid"` - SupportedAuth []string `msgpack:"supported_auth"` - Version string `msgpack:"version"` - RealRdma bool `msgpack:"real_rdma"` -} - -// usize corresponds to Rust's usize type (platform dependent, but typically uint64 on 64-bit systems) -type usize uint64 - -// PingRequest corresponds to Rust PingRequest -type PingRequest struct { - TimestampNs uint64 `msgpack:"timestamp_ns"` - ClientID *string `msgpack:"client_id,omitempty"` -} - -// PongResponse corresponds to Rust PongResponse -type PongResponse struct { - ClientTimestampNs uint64 `msgpack:"client_timestamp_ns"` - ServerTimestampNs uint64 `msgpack:"server_timestamp_ns"` - ServerRttNs uint64 `msgpack:"server_rtt_ns"` -} - -// ErrorResponse corresponds to Rust ErrorResponse -type ErrorResponse struct { - Code string `msgpack:"code"` - Message string `msgpack:"message"` - Details *string `msgpack:"details,omitempty"` -} - -// Helper functions for creating messages -func NewStartReadMessage(req *StartReadRequest) *IpcMessage { - return &IpcMessage{ - Type: MsgStartRead, - Data: req, - } -} - -func NewCompleteReadMessage(sessionID string, success bool, bytesTransferred uint64, clientCrc *uint32, errorMessage *string) *IpcMessage { - return &IpcMessage{ - Type: MsgCompleteRead, - Data: &CompleteReadRequest{ - SessionID: sessionID, - Success: success, - BytesTransferred: bytesTransferred, - ClientCrc: clientCrc, - ErrorMessage: errorMessage, - }, - } -} - -func NewGetCapabilitiesMessage(clientID *string) *IpcMessage { - return &IpcMessage{ - Type: MsgGetCapabilities, - Data: &GetCapabilitiesRequest{ - ClientID: clientID, - }, - } -} - -func NewPingMessage(clientID *string) *IpcMessage { - return &IpcMessage{ - Type: MsgPing, - Data: &PingRequest{ - TimestampNs: uint64(time.Now().UnixNano()), - ClientID: clientID, - }, - } -} - -func NewErrorMessage(code, message string, details *string) *IpcMessage { - return &IpcMessage{ - Type: MsgError, - Data: &ErrorResponse{ - Code: code, - Message: message, - Details: details, - }, - } -} diff --git a/seaweedfs-rdma-sidecar/pkg/rdma/client.go b/seaweedfs-rdma-sidecar/pkg/rdma/client.go deleted file mode 100644 index 08ad76042..000000000 --- a/seaweedfs-rdma-sidecar/pkg/rdma/client.go +++ /dev/null @@ -1,630 +0,0 @@ -// Package rdma provides high-level RDMA operations for SeaweedFS integration -package rdma - -import ( - "context" - "fmt" - "sync" - "time" - - "seaweedfs-rdma-sidecar/pkg/ipc" - - "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "github.com/sirupsen/logrus" -) - -// PooledConnection represents a pooled RDMA connection -type PooledConnection struct { - ipcClient *ipc.Client - lastUsed time.Time - inUse bool - sessionID string - created time.Time -} - -// ConnectionPool manages a pool of RDMA connections -type ConnectionPool struct { - connections []*PooledConnection - mutex sync.RWMutex - maxConnections int - maxIdleTime time.Duration - enginePath string - logger *logrus.Logger -} - -// Client provides high-level RDMA operations with connection pooling -type Client struct { - pool *ConnectionPool - logger *logrus.Logger - enginePath string - capabilities *ipc.GetCapabilitiesResponse - connected bool - defaultTimeout time.Duration - - // Legacy single connection (for backward compatibility) - ipcClient *ipc.Client -} - -// Config holds configuration for the RDMA client -type Config struct { - EngineSocketPath string - DefaultTimeout time.Duration - Logger *logrus.Logger - - // Connection pooling options - EnablePooling bool // Enable connection pooling (default: true) - MaxConnections int // Max connections in pool (default: 10) - MaxIdleTime time.Duration // Max idle time before connection cleanup (default: 5min) -} - -// ReadRequest represents a SeaweedFS needle read request -type ReadRequest struct { - VolumeID uint32 - NeedleID uint64 - Cookie uint32 - Offset uint64 - Size uint64 - AuthToken *string -} - -// ReadResponse represents the result of an RDMA read operation -type ReadResponse struct { - Data []byte - BytesRead uint64 - Duration time.Duration - TransferRate float64 - SessionID string - Success bool - Message string -} - -// NewConnectionPool creates a new connection pool -func NewConnectionPool(enginePath string, maxConnections int, maxIdleTime time.Duration, logger *logrus.Logger) *ConnectionPool { - if maxConnections <= 0 { - maxConnections = 10 // Default - } - if maxIdleTime <= 0 { - maxIdleTime = 5 * time.Minute // Default - } - - return &ConnectionPool{ - connections: make([]*PooledConnection, 0, maxConnections), - maxConnections: maxConnections, - maxIdleTime: maxIdleTime, - enginePath: enginePath, - logger: logger, - } -} - -// getConnection gets an available connection from the pool or creates a new one -func (p *ConnectionPool) getConnection(ctx context.Context) (*PooledConnection, error) { - p.mutex.Lock() - defer p.mutex.Unlock() - - // Look for an available connection - for _, conn := range p.connections { - if !conn.inUse && time.Since(conn.lastUsed) < p.maxIdleTime { - conn.inUse = true - conn.lastUsed = time.Now() - p.logger.WithField("session_id", conn.sessionID).Debug("๐Ÿ”Œ Reusing pooled RDMA connection") - return conn, nil - } - } - - // Create new connection if under limit - if len(p.connections) < p.maxConnections { - ipcClient := ipc.NewClient(p.enginePath, p.logger) - if err := ipcClient.Connect(ctx); err != nil { - return nil, fmt.Errorf("failed to create new pooled connection: %w", err) - } - - conn := &PooledConnection{ - ipcClient: ipcClient, - lastUsed: time.Now(), - inUse: true, - sessionID: fmt.Sprintf("pool-%d-%d", len(p.connections), time.Now().Unix()), - created: time.Now(), - } - - p.connections = append(p.connections, conn) - p.logger.WithFields(logrus.Fields{ - "session_id": conn.sessionID, - "pool_size": len(p.connections), - }).Info("๐Ÿš€ Created new pooled RDMA connection") - - return conn, nil - } - - // Pool is full, wait for an available connection - return nil, fmt.Errorf("connection pool exhausted (max: %d)", p.maxConnections) -} - -// releaseConnection returns a connection to the pool -func (p *ConnectionPool) releaseConnection(conn *PooledConnection) { - p.mutex.Lock() - defer p.mutex.Unlock() - - conn.inUse = false - conn.lastUsed = time.Now() - - p.logger.WithField("session_id", conn.sessionID).Debug("๐Ÿ”„ Released RDMA connection back to pool") -} - -// cleanup removes idle connections from the pool -func (p *ConnectionPool) cleanup() { - p.mutex.Lock() - defer p.mutex.Unlock() - - now := time.Now() - activeConnections := make([]*PooledConnection, 0, len(p.connections)) - - for _, conn := range p.connections { - if conn.inUse || now.Sub(conn.lastUsed) < p.maxIdleTime { - activeConnections = append(activeConnections, conn) - } else { - // Close idle connection - conn.ipcClient.Disconnect() - p.logger.WithFields(logrus.Fields{ - "session_id": conn.sessionID, - "idle_time": now.Sub(conn.lastUsed), - }).Debug("๐Ÿงน Cleaned up idle RDMA connection") - } - } - - p.connections = activeConnections -} - -// Close closes all connections in the pool -func (p *ConnectionPool) Close() { - p.mutex.Lock() - defer p.mutex.Unlock() - - for _, conn := range p.connections { - conn.ipcClient.Disconnect() - } - p.connections = nil - p.logger.Info("๐Ÿ”Œ Connection pool closed") -} - -// NewClient creates a new RDMA client -func NewClient(config *Config) *Client { - if config.Logger == nil { - config.Logger = logrus.New() - config.Logger.SetLevel(logrus.InfoLevel) - } - - if config.DefaultTimeout == 0 { - config.DefaultTimeout = 30 * time.Second - } - - client := &Client{ - logger: config.Logger, - enginePath: config.EngineSocketPath, - defaultTimeout: config.DefaultTimeout, - } - - // Initialize connection pooling if enabled (default: true) - enablePooling := config.EnablePooling - if config.MaxConnections == 0 && config.MaxIdleTime == 0 { - // Default to enabled if not explicitly configured - enablePooling = true - } - - if enablePooling { - client.pool = NewConnectionPool( - config.EngineSocketPath, - config.MaxConnections, - config.MaxIdleTime, - config.Logger, - ) - - // Start cleanup goroutine - go client.startCleanupRoutine() - - config.Logger.WithFields(logrus.Fields{ - "max_connections": client.pool.maxConnections, - "max_idle_time": client.pool.maxIdleTime, - }).Info("๐Ÿ”Œ RDMA connection pooling enabled") - } else { - // Legacy single connection mode - client.ipcClient = ipc.NewClient(config.EngineSocketPath, config.Logger) - config.Logger.Info("๐Ÿ”Œ RDMA single connection mode (pooling disabled)") - } - - return client -} - -// startCleanupRoutine starts a background goroutine to clean up idle connections -func (c *Client) startCleanupRoutine() { - ticker := time.NewTicker(1 * time.Minute) // Cleanup every minute - go func() { - defer ticker.Stop() - for range ticker.C { - if c.pool != nil { - c.pool.cleanup() - } - } - }() -} - -// Connect establishes connection to the Rust RDMA engine and queries capabilities -func (c *Client) Connect(ctx context.Context) error { - c.logger.Info("๐Ÿš€ Connecting to RDMA engine") - - if c.pool != nil { - // Connection pooling mode - connections are created on-demand - c.connected = true - c.logger.Info("โœ… RDMA client ready (connection pooling enabled)") - return nil - } - - // Single connection mode - if err := c.ipcClient.Connect(ctx); err != nil { - return fmt.Errorf("failed to connect to IPC: %w", err) - } - - // Test connectivity with ping - clientID := "rdma-client" - pong, err := c.ipcClient.Ping(ctx, &clientID) - if err != nil { - c.ipcClient.Disconnect() - return fmt.Errorf("failed to ping RDMA engine: %w", err) - } - - latency := time.Duration(pong.ServerRttNs) - c.logger.WithFields(logrus.Fields{ - "latency": latency, - "server_rtt": time.Duration(pong.ServerRttNs), - }).Info("๐Ÿ“ก RDMA engine ping successful") - - // Get capabilities - caps, err := c.ipcClient.GetCapabilities(ctx, &clientID) - if err != nil { - c.ipcClient.Disconnect() - return fmt.Errorf("failed to get engine capabilities: %w", err) - } - - c.capabilities = caps - c.connected = true - - c.logger.WithFields(logrus.Fields{ - "version": caps.Version, - "device_name": caps.DeviceName, - "vendor_id": caps.VendorId, - "max_sessions": caps.MaxSessions, - "max_transfer_size": caps.MaxTransferSize, - "active_sessions": caps.ActiveSessions, - "real_rdma": caps.RealRdma, - "port_gid": caps.PortGid, - "port_lid": caps.PortLid, - }).Info("โœ… RDMA engine connected and ready") - - return nil -} - -// Disconnect closes the connection to the RDMA engine -func (c *Client) Disconnect() { - if c.connected { - if c.pool != nil { - // Connection pooling mode - c.pool.Close() - c.logger.Info("๐Ÿ”Œ Disconnected from RDMA engine (pool closed)") - } else { - // Single connection mode - c.ipcClient.Disconnect() - c.logger.Info("๐Ÿ”Œ Disconnected from RDMA engine") - } - c.connected = false - } -} - -// IsConnected returns true if connected to the RDMA engine -func (c *Client) IsConnected() bool { - if c.pool != nil { - // Connection pooling mode - always connected if pool exists - return c.connected - } else { - // Single connection mode - return c.connected && c.ipcClient.IsConnected() - } -} - -// GetCapabilities returns the RDMA engine capabilities -func (c *Client) GetCapabilities() *ipc.GetCapabilitiesResponse { - return c.capabilities -} - -// Read performs an RDMA read operation for a SeaweedFS needle -func (c *Client) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, error) { - if !c.IsConnected() { - return nil, fmt.Errorf("not connected to RDMA engine") - } - - startTime := time.Now() - - c.logger.WithFields(logrus.Fields{ - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - "offset": req.Offset, - "size": req.Size, - }).Debug("๐Ÿ“– Starting RDMA read operation") - - if c.pool != nil { - // Connection pooling mode - return c.readWithPool(ctx, req, startTime) - } - - // Single connection mode - // Create IPC request - ipcReq := &ipc.StartReadRequest{ - VolumeID: req.VolumeID, - NeedleID: req.NeedleID, - Cookie: req.Cookie, - Offset: req.Offset, - Size: req.Size, - RemoteAddr: 0, // Will be set by engine (mock for now) - RemoteKey: 0, // Will be set by engine (mock for now) - TimeoutSecs: uint64(c.defaultTimeout.Seconds()), - AuthToken: req.AuthToken, - } - - // Start RDMA read - startResp, err := c.ipcClient.StartRead(ctx, ipcReq) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to start RDMA read") - return nil, fmt.Errorf("failed to start RDMA read: %w", err) - } - - // In the new protocol, if we got a StartReadResponse, the operation was successful - - c.logger.WithFields(logrus.Fields{ - "session_id": startResp.SessionID, - "local_addr": fmt.Sprintf("0x%x", startResp.LocalAddr), - "local_key": startResp.LocalKey, - "transfer_size": startResp.TransferSize, - "expected_crc": fmt.Sprintf("0x%x", startResp.ExpectedCrc), - "expires_at": time.Unix(0, int64(startResp.ExpiresAtNs)).Format(time.RFC3339), - }).Debug("๐Ÿ“– RDMA read session started") - - // Complete the RDMA read - completeResp, err := c.ipcClient.CompleteRead(ctx, startResp.SessionID, true, startResp.TransferSize, &startResp.ExpectedCrc) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to complete RDMA read") - return nil, fmt.Errorf("failed to complete RDMA read: %w", err) - } - - duration := time.Since(startTime) - - if !completeResp.Success { - errorMsg := "unknown error" - if completeResp.Message != nil { - errorMsg = *completeResp.Message - } - c.logger.WithFields(logrus.Fields{ - "session_id": startResp.SessionID, - "error_message": errorMsg, - }).Error("โŒ RDMA read completion failed") - return nil, fmt.Errorf("RDMA read completion failed: %s", errorMsg) - } - - // Calculate transfer rate (bytes/second) - transferRate := float64(startResp.TransferSize) / duration.Seconds() - - c.logger.WithFields(logrus.Fields{ - "session_id": startResp.SessionID, - "bytes_read": startResp.TransferSize, - "duration": duration, - "transfer_rate": transferRate, - "server_crc": completeResp.ServerCrc, - }).Info("โœ… RDMA read completed successfully") - - // MOCK DATA IMPLEMENTATION - FOR DEVELOPMENT/TESTING ONLY - // - // This section generates placeholder data for the mock RDMA implementation. - // In a production RDMA implementation, this should be replaced with: - // - // 1. The actual data transferred via RDMA from the remote memory region - // 2. Data validation using checksums/CRC from the RDMA completion - // 3. Proper error handling for RDMA transfer failures - // 4. Memory region cleanup and deregistration - // - // TODO for real RDMA implementation: - // - Replace mockData with actual RDMA buffer contents - // - Validate data integrity using server CRC: completeResp.ServerCrc - // - Handle partial transfers and retry logic - // - Implement proper memory management for RDMA regions - // - // Current mock behavior: Generates a simple pattern (0,1,2...255,0,1,2...) - // This allows testing of the integration pipeline without real hardware - mockData := make([]byte, startResp.TransferSize) - for i := range mockData { - mockData[i] = byte(i % 256) // Simple repeating pattern for verification - } - // END MOCK DATA IMPLEMENTATION - - return &ReadResponse{ - Data: mockData, - BytesRead: startResp.TransferSize, - Duration: duration, - TransferRate: transferRate, - SessionID: startResp.SessionID, - Success: true, - Message: "RDMA read completed successfully", - }, nil -} - -// ReadRange performs an RDMA read for a specific range within a needle -func (c *Client) ReadRange(ctx context.Context, volumeID uint32, needleID uint64, cookie uint32, offset, size uint64) (*ReadResponse, error) { - req := &ReadRequest{ - VolumeID: volumeID, - NeedleID: needleID, - Cookie: cookie, - Offset: offset, - Size: size, - } - return c.Read(ctx, req) -} - -// ReadFileRange performs an RDMA read using SeaweedFS file ID format -func (c *Client) ReadFileRange(ctx context.Context, fileID string, offset, size uint64) (*ReadResponse, error) { - // Parse file ID (e.g., "3,01637037d6" -> volume=3, needle=0x01637037d6, cookie extracted) - volumeID, needleID, cookie, err := parseFileID(fileID) - if err != nil { - return nil, fmt.Errorf("invalid file ID %s: %w", fileID, err) - } - - req := &ReadRequest{ - VolumeID: volumeID, - NeedleID: needleID, - Cookie: cookie, - Offset: offset, - Size: size, - } - return c.Read(ctx, req) -} - -// parseFileID extracts volume ID, needle ID, and cookie from a SeaweedFS file ID -// Uses existing SeaweedFS parsing logic to ensure compatibility -func parseFileID(fileId string) (volumeID uint32, needleID uint64, cookie uint32, err error) { - // Use existing SeaweedFS file ID parsing - fid, err := needle.ParseFileIdFromString(fileId) - if err != nil { - return 0, 0, 0, fmt.Errorf("failed to parse file ID %s: %w", fileId, err) - } - - volumeID = uint32(fid.VolumeId) - needleID = uint64(fid.Key) - cookie = uint32(fid.Cookie) - - return volumeID, needleID, cookie, nil -} - -// ReadFull performs an RDMA read for an entire needle -func (c *Client) ReadFull(ctx context.Context, volumeID uint32, needleID uint64, cookie uint32) (*ReadResponse, error) { - req := &ReadRequest{ - VolumeID: volumeID, - NeedleID: needleID, - Cookie: cookie, - Offset: 0, - Size: 0, // 0 means read entire needle - } - return c.Read(ctx, req) -} - -// Ping tests connectivity to the RDMA engine -func (c *Client) Ping(ctx context.Context) (time.Duration, error) { - if !c.IsConnected() { - return 0, fmt.Errorf("not connected to RDMA engine") - } - - clientID := "health-check" - start := time.Now() - pong, err := c.ipcClient.Ping(ctx, &clientID) - if err != nil { - return 0, err - } - - totalLatency := time.Since(start) - serverRtt := time.Duration(pong.ServerRttNs) - - c.logger.WithFields(logrus.Fields{ - "total_latency": totalLatency, - "server_rtt": serverRtt, - "client_id": clientID, - }).Debug("๐Ÿ“ RDMA engine ping successful") - - return totalLatency, nil -} - -// readWithPool performs RDMA read using connection pooling -func (c *Client) readWithPool(ctx context.Context, req *ReadRequest, startTime time.Time) (*ReadResponse, error) { - // Get connection from pool - conn, err := c.pool.getConnection(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get pooled connection: %w", err) - } - defer c.pool.releaseConnection(conn) - - c.logger.WithField("session_id", conn.sessionID).Debug("๐Ÿ”Œ Using pooled RDMA connection") - - // Create IPC request - ipcReq := &ipc.StartReadRequest{ - VolumeID: req.VolumeID, - NeedleID: req.NeedleID, - Cookie: req.Cookie, - Offset: req.Offset, - Size: req.Size, - RemoteAddr: 0, // Will be set by engine (mock for now) - RemoteKey: 0, // Will be set by engine (mock for now) - TimeoutSecs: uint64(c.defaultTimeout.Seconds()), - AuthToken: req.AuthToken, - } - - // Start RDMA read - startResp, err := conn.ipcClient.StartRead(ctx, ipcReq) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to start RDMA read (pooled)") - return nil, fmt.Errorf("failed to start RDMA read: %w", err) - } - - c.logger.WithFields(logrus.Fields{ - "session_id": startResp.SessionID, - "local_addr": fmt.Sprintf("0x%x", startResp.LocalAddr), - "local_key": startResp.LocalKey, - "transfer_size": startResp.TransferSize, - "expected_crc": fmt.Sprintf("0x%x", startResp.ExpectedCrc), - "expires_at": time.Unix(0, int64(startResp.ExpiresAtNs)).Format(time.RFC3339), - "pooled": true, - }).Debug("๐Ÿ“– RDMA read session started (pooled)") - - // Complete the RDMA read - completeResp, err := conn.ipcClient.CompleteRead(ctx, startResp.SessionID, true, startResp.TransferSize, &startResp.ExpectedCrc) - if err != nil { - c.logger.WithError(err).Error("โŒ Failed to complete RDMA read (pooled)") - return nil, fmt.Errorf("failed to complete RDMA read: %w", err) - } - - duration := time.Since(startTime) - - if !completeResp.Success { - errorMsg := "unknown error" - if completeResp.Message != nil { - errorMsg = *completeResp.Message - } - c.logger.WithFields(logrus.Fields{ - "session_id": conn.sessionID, - "error_message": errorMsg, - "pooled": true, - }).Error("โŒ RDMA read completion failed (pooled)") - return nil, fmt.Errorf("RDMA read completion failed: %s", errorMsg) - } - - // Calculate transfer rate (bytes/second) - transferRate := float64(startResp.TransferSize) / duration.Seconds() - - c.logger.WithFields(logrus.Fields{ - "session_id": conn.sessionID, - "bytes_read": startResp.TransferSize, - "duration": duration, - "transfer_rate": transferRate, - "server_crc": completeResp.ServerCrc, - "pooled": true, - }).Info("โœ… RDMA read completed successfully (pooled)") - - // For the mock implementation, we'll return placeholder data - // In the real implementation, this would be the actual RDMA transferred data - mockData := make([]byte, startResp.TransferSize) - for i := range mockData { - mockData[i] = byte(i % 256) // Simple pattern for testing - } - - return &ReadResponse{ - Data: mockData, - BytesRead: startResp.TransferSize, - Duration: duration, - TransferRate: transferRate, - SessionID: conn.sessionID, - Success: true, - Message: "RDMA read successful (pooled)", - }, nil -} diff --git a/seaweedfs-rdma-sidecar/pkg/seaweedfs/client.go b/seaweedfs-rdma-sidecar/pkg/seaweedfs/client.go deleted file mode 100644 index 22047e504..000000000 --- a/seaweedfs-rdma-sidecar/pkg/seaweedfs/client.go +++ /dev/null @@ -1,402 +0,0 @@ -// Package seaweedfs provides SeaweedFS-specific RDMA integration -package seaweedfs - -import ( - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "seaweedfs-rdma-sidecar/pkg/rdma" - - "github.com/seaweedfs/seaweedfs/weed/storage/needle" - "github.com/seaweedfs/seaweedfs/weed/storage/types" - "github.com/sirupsen/logrus" -) - -// SeaweedFSRDMAClient provides SeaweedFS-specific RDMA operations -type SeaweedFSRDMAClient struct { - rdmaClient *rdma.Client - logger *logrus.Logger - volumeServerURL string - enabled bool - - // Zero-copy optimization - tempDir string - useZeroCopy bool -} - -// Config holds configuration for the SeaweedFS RDMA client -type Config struct { - RDMASocketPath string - VolumeServerURL string - Enabled bool - DefaultTimeout time.Duration - Logger *logrus.Logger - - // Zero-copy optimization - TempDir string // Directory for temp files (default: /tmp/rdma-cache) - UseZeroCopy bool // Enable zero-copy via temp files - - // Connection pooling options - EnablePooling bool // Enable RDMA connection pooling (default: true) - MaxConnections int // Max connections in pool (default: 10) - MaxIdleTime time.Duration // Max idle time before connection cleanup (default: 5min) -} - -// NeedleReadRequest represents a SeaweedFS needle read request -type NeedleReadRequest struct { - VolumeID uint32 - NeedleID uint64 - Cookie uint32 - Offset uint64 - Size uint64 - VolumeServer string // Override volume server URL for this request -} - -// NeedleReadResponse represents the result of a needle read -type NeedleReadResponse struct { - Data []byte - IsRDMA bool - Latency time.Duration - Source string // "rdma" or "http" - SessionID string - - // Zero-copy optimization fields - TempFilePath string // Path to temp file with data (for zero-copy) - UseTempFile bool // Whether to use temp file instead of Data -} - -// NewSeaweedFSRDMAClient creates a new SeaweedFS RDMA client -func NewSeaweedFSRDMAClient(config *Config) (*SeaweedFSRDMAClient, error) { - if config.Logger == nil { - config.Logger = logrus.New() - config.Logger.SetLevel(logrus.InfoLevel) - } - - var rdmaClient *rdma.Client - if config.Enabled && config.RDMASocketPath != "" { - rdmaConfig := &rdma.Config{ - EngineSocketPath: config.RDMASocketPath, - DefaultTimeout: config.DefaultTimeout, - Logger: config.Logger, - EnablePooling: config.EnablePooling, - MaxConnections: config.MaxConnections, - MaxIdleTime: config.MaxIdleTime, - } - rdmaClient = rdma.NewClient(rdmaConfig) - } - - // Setup temp directory for zero-copy optimization - tempDir := config.TempDir - if tempDir == "" { - tempDir = "/tmp/rdma-cache" - } - - if config.UseZeroCopy { - if err := os.MkdirAll(tempDir, 0755); err != nil { - config.Logger.WithError(err).Warn("Failed to create temp directory, disabling zero-copy") - config.UseZeroCopy = false - } - } - - return &SeaweedFSRDMAClient{ - rdmaClient: rdmaClient, - logger: config.Logger, - volumeServerURL: config.VolumeServerURL, - enabled: config.Enabled, - tempDir: tempDir, - useZeroCopy: config.UseZeroCopy, - }, nil -} - -// Start initializes the RDMA client connection -func (c *SeaweedFSRDMAClient) Start(ctx context.Context) error { - if !c.enabled || c.rdmaClient == nil { - c.logger.Info("๐Ÿ”„ RDMA disabled, using HTTP fallback only") - return nil - } - - c.logger.Info("๐Ÿš€ Starting SeaweedFS RDMA client...") - - if err := c.rdmaClient.Connect(ctx); err != nil { - c.logger.WithError(err).Error("โŒ Failed to connect to RDMA engine") - return fmt.Errorf("failed to connect to RDMA engine: %w", err) - } - - c.logger.Info("โœ… SeaweedFS RDMA client started successfully") - return nil -} - -// Stop shuts down the RDMA client -func (c *SeaweedFSRDMAClient) Stop() { - if c.rdmaClient != nil { - c.rdmaClient.Disconnect() - c.logger.Info("๐Ÿ”Œ SeaweedFS RDMA client stopped") - } -} - -// IsEnabled returns true if RDMA is enabled and available -func (c *SeaweedFSRDMAClient) IsEnabled() bool { - return c.enabled && c.rdmaClient != nil && c.rdmaClient.IsConnected() -} - -// ReadNeedle reads a needle using RDMA fast path or HTTP fallback -func (c *SeaweedFSRDMAClient) ReadNeedle(ctx context.Context, req *NeedleReadRequest) (*NeedleReadResponse, error) { - start := time.Now() - var rdmaErr error - - // Try RDMA fast path first - if c.IsEnabled() { - c.logger.WithFields(logrus.Fields{ - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - "offset": req.Offset, - "size": req.Size, - }).Debug("๐Ÿš€ Attempting RDMA fast path") - - rdmaReq := &rdma.ReadRequest{ - VolumeID: req.VolumeID, - NeedleID: req.NeedleID, - Cookie: req.Cookie, - Offset: req.Offset, - Size: req.Size, - } - - resp, err := c.rdmaClient.Read(ctx, rdmaReq) - if err != nil { - c.logger.WithError(err).Warn("โš ๏ธ RDMA read failed, falling back to HTTP") - rdmaErr = err - } else { - c.logger.WithFields(logrus.Fields{ - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - "bytes_read": resp.BytesRead, - "transfer_rate": resp.TransferRate, - "latency": time.Since(start), - }).Info("๐Ÿš€ RDMA fast path successful") - - // Try zero-copy optimization if enabled and data is large enough - if c.useZeroCopy && len(resp.Data) > 64*1024 { // 64KB threshold - tempFilePath, err := c.writeToTempFile(req, resp.Data) - if err != nil { - c.logger.WithError(err).Warn("Failed to write temp file, using regular response") - // Fall back to regular response - } else { - c.logger.WithFields(logrus.Fields{ - "temp_file": tempFilePath, - "size": len(resp.Data), - }).Info("๐Ÿ”ฅ Zero-copy temp file created") - - return &NeedleReadResponse{ - Data: nil, // Don't duplicate data in memory - IsRDMA: true, - Latency: time.Since(start), - Source: "rdma-zerocopy", - SessionID: resp.SessionID, - TempFilePath: tempFilePath, - UseTempFile: true, - }, nil - } - } - - return &NeedleReadResponse{ - Data: resp.Data, - IsRDMA: true, - Latency: time.Since(start), - Source: "rdma", - SessionID: resp.SessionID, - }, nil - } - } - - // Fallback to HTTP - c.logger.WithFields(logrus.Fields{ - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - "reason": "rdma_unavailable", - }).Debug("๐ŸŒ Using HTTP fallback") - - data, err := c.httpFallback(ctx, req) - if err != nil { - if rdmaErr != nil { - return nil, fmt.Errorf("both RDMA and HTTP fallback failed: RDMA=%v, HTTP=%v", rdmaErr, err) - } - return nil, fmt.Errorf("HTTP fallback failed: %w", err) - } - - return &NeedleReadResponse{ - Data: data, - IsRDMA: false, - Latency: time.Since(start), - Source: "http", - }, nil -} - -// ReadNeedleRange reads a specific range from a needle -func (c *SeaweedFSRDMAClient) ReadNeedleRange(ctx context.Context, volumeID uint32, needleID uint64, cookie uint32, offset, size uint64) (*NeedleReadResponse, error) { - req := &NeedleReadRequest{ - VolumeID: volumeID, - NeedleID: needleID, - Cookie: cookie, - Offset: offset, - Size: size, - } - return c.ReadNeedle(ctx, req) -} - -// httpFallback performs HTTP fallback read from SeaweedFS volume server -func (c *SeaweedFSRDMAClient) httpFallback(ctx context.Context, req *NeedleReadRequest) ([]byte, error) { - // Use volume server from request, fallback to configured URL - volumeServerURL := req.VolumeServer - if volumeServerURL == "" { - volumeServerURL = c.volumeServerURL - } - - if volumeServerURL == "" { - return nil, fmt.Errorf("no volume server URL provided in request or configured") - } - - // Build URL using existing SeaweedFS file ID construction - volumeId := needle.VolumeId(req.VolumeID) - needleId := types.NeedleId(req.NeedleID) - cookie := types.Cookie(req.Cookie) - - fileId := &needle.FileId{ - VolumeId: volumeId, - Key: needleId, - Cookie: cookie, - } - - url := fmt.Sprintf("%s/%s", volumeServerURL, fileId.String()) - - if req.Offset > 0 || req.Size > 0 { - url += fmt.Sprintf("?offset=%d&size=%d", req.Offset, req.Size) - } - - c.logger.WithField("url", url).Debug("๐Ÿ“ฅ HTTP fallback request") - - httpReq, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return nil, fmt.Errorf("failed to create HTTP request: %w", err) - } - - client := &http.Client{Timeout: 30 * time.Second} - resp, err := client.Do(httpReq) - if err != nil { - return nil, fmt.Errorf("HTTP request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("HTTP request failed with status: %d", resp.StatusCode) - } - - // Read response data - io.ReadAll handles context cancellation and timeouts correctly - data, err := io.ReadAll(resp.Body) - if err != nil { - return nil, fmt.Errorf("failed to read HTTP response body: %w", err) - } - - c.logger.WithFields(logrus.Fields{ - "volume_id": req.VolumeID, - "needle_id": req.NeedleID, - "data_size": len(data), - }).Debug("๐Ÿ“ฅ HTTP fallback successful") - - return data, nil -} - -// HealthCheck verifies that the RDMA client is healthy -func (c *SeaweedFSRDMAClient) HealthCheck(ctx context.Context) error { - if !c.enabled { - return fmt.Errorf("RDMA is disabled") - } - - if c.rdmaClient == nil { - return fmt.Errorf("RDMA client not initialized") - } - - if !c.rdmaClient.IsConnected() { - return fmt.Errorf("RDMA client not connected") - } - - // Try a ping to the RDMA engine - _, err := c.rdmaClient.Ping(ctx) - return err -} - -// GetStats returns statistics about the RDMA client -func (c *SeaweedFSRDMAClient) GetStats() map[string]interface{} { - stats := map[string]interface{}{ - "enabled": c.enabled, - "volume_server_url": c.volumeServerURL, - "rdma_socket_path": "", - } - - if c.rdmaClient != nil { - stats["connected"] = c.rdmaClient.IsConnected() - // Note: Capabilities method may not be available, skip for now - } else { - stats["connected"] = false - stats["error"] = "RDMA client not initialized" - } - - return stats -} - -// writeToTempFile writes RDMA data to a temp file for zero-copy optimization -func (c *SeaweedFSRDMAClient) writeToTempFile(req *NeedleReadRequest, data []byte) (string, error) { - // Create temp file with unique name based on needle info - fileName := fmt.Sprintf("vol%d_needle%x_cookie%d_offset%d_size%d.tmp", - req.VolumeID, req.NeedleID, req.Cookie, req.Offset, req.Size) - tempFilePath := filepath.Join(c.tempDir, fileName) - - // Write data to temp file (this populates the page cache) - err := os.WriteFile(tempFilePath, data, 0644) - if err != nil { - return "", fmt.Errorf("failed to write temp file: %w", err) - } - - c.logger.WithFields(logrus.Fields{ - "temp_file": tempFilePath, - "size": len(data), - }).Debug("๐Ÿ“ Temp file written to page cache") - - return tempFilePath, nil -} - -// CleanupTempFile removes a temp file (called by mount client after use) -func (c *SeaweedFSRDMAClient) CleanupTempFile(tempFilePath string) error { - if tempFilePath == "" { - return nil - } - - // Validate that tempFilePath is within c.tempDir - absTempDir, err := filepath.Abs(c.tempDir) - if err != nil { - return fmt.Errorf("failed to resolve temp dir: %w", err) - } - absFilePath, err := filepath.Abs(tempFilePath) - if err != nil { - return fmt.Errorf("failed to resolve temp file path: %w", err) - } - // Ensure absFilePath is within absTempDir - if !strings.HasPrefix(absFilePath, absTempDir+string(os.PathSeparator)) && absFilePath != absTempDir { - c.logger.WithField("temp_file", tempFilePath).Warn("Attempted cleanup of file outside temp dir") - return fmt.Errorf("invalid temp file path") - } - - err = os.Remove(absFilePath) - if err != nil && !os.IsNotExist(err) { - c.logger.WithError(err).WithField("temp_file", absFilePath).Warn("Failed to cleanup temp file") - return err - } - - c.logger.WithField("temp_file", absFilePath).Debug("๐Ÿงน Temp file cleaned up") - return nil -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock deleted file mode 100644 index 6c2b85197..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.lock +++ /dev/null @@ -1,1934 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.24.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "ahash" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" -dependencies = [ - "getrandom 0.2.16", - "once_cell", - "version_check", -] - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - -[[package]] -name = "anstream" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" - -[[package]] -name = "anstyle-parse" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2" -dependencies = [ - "windows-sys 0.60.2", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a" -dependencies = [ - "anstyle", - "once_cell_polyfill", - "windows-sys 0.60.2", -] - -[[package]] -name = "anyhow" -version = "1.0.99" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "backtrace" -version = "0.3.75" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-targets 0.52.6", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a65b545ab31d687cff52899d4890855fec459eb6afe0da6417b8a18da87aa29" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "bytes" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" - -[[package]] -name = "cast" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" - -[[package]] -name = "cc" -version = "1.2.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" -dependencies = [ - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" - -[[package]] -name = "chrono" -version = "0.4.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "serde", - "wasm-bindgen", - "windows-link", -] - -[[package]] -name = "ciborium" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" - -[[package]] -name = "ciborium-ll" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" -dependencies = [ - "ciborium-io", - "half", -] - -[[package]] -name = "clap" -version = "4.5.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc0e74a703892159f5ae7d3aac52c8e6c392f5ae5f359c70b5881d60aaac318" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3e7f4214277f3c7aa526a59dd3fbe306a370daee1f8b7b8c987069cd8e888a8" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14cb31bb0a7d536caef2639baa7fad459e15c3144efefa6dbd1c84562c4739f6" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "clap_lex" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" - -[[package]] -name = "colorchoice" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" - -[[package]] -name = "config" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23738e11972c7643e4ec947840fc463b6a571afcd3e735bdfce7d03c7a784aca" -dependencies = [ - "async-trait", - "json5", - "lazy_static", - "nom", - "pathdiff", - "ron", - "rust-ini", - "serde", - "serde_json", - "toml", - "yaml-rust", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "criterion" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" -dependencies = [ - "anes", - "cast", - "ciborium", - "clap", - "criterion-plot", - "is-terminal", - "itertools", - "num-traits", - "once_cell", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "walkdir", -] - -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "dlv-list" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "errno" -version = "0.3.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" -dependencies = [ - "libc", - "windows-sys 0.60.2", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.2+wasi-0.2.4", -] - -[[package]] -name = "gimli" -version = "0.31.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" - -[[package]] -name = "half" -version = "2.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" -dependencies = [ - "cfg-if", - "crunchy", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" - -[[package]] -name = "iana-time-zone" -version = "0.1.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "log", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "io-uring" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" -dependencies = [ - "bitflags 2.9.2", - "cfg-if", - "libc", -] - -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "js-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "json5" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" -dependencies = [ - "pest", - "pest_derive", - "serde", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.175" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" - -[[package]] -name = "libloading" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" -dependencies = [ - "cfg-if", - "windows-targets 0.53.3", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - -[[package]] -name = "linux-raw-sys" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" - -[[package]] -name = "lock_api" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - -[[package]] -name = "matchers" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" -dependencies = [ - "regex-automata", -] - -[[package]] -name = "memchr" -version = "2.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" - -[[package]] -name = "memmap2" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" -dependencies = [ - "libc", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.9.2", - "cfg-if", - "libc", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "nu-ansi-term" -version = "0.50.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "object" -version = "0.36.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "once_cell_polyfill" -version = "1.70.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" - -[[package]] -name = "oorandom" -version = "11.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" - -[[package]] -name = "ordered-multimap" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd746e37177e1711c20dd619a1620f34f5c8b569c53590a72dedd5344d8924a" -dependencies = [ - "dlv-list", - "hashbrown", -] - -[[package]] -name = "parking_lot" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets 0.52.6", -] - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - -[[package]] -name = "pest" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db05f56d34358a8b1066f67cbb203ee3e7ed2ba674a6263a1d5ec6db2204323" -dependencies = [ - "memchr", - "thiserror 2.0.15", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb056d9e8ea77922845ec74a1c4e8fb17e7c218cc4fc11a15c5d25e189aa40bc" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87e404e638f781eb3202dc82db6760c8ae8a1eeef7fb3fa8264b2ef280504966" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pest_meta" -version = "2.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd1101f170f5903fde0914f899bb503d9ff5271d7ba76bbb70bea63690cc0d5" -dependencies = [ - "pest", - "sha2", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "plotters" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" -dependencies = [ - "num-traits", - "plotters-backend", - "plotters-svg", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "plotters-backend" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" - -[[package]] -name = "plotters-svg" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" -dependencies = [ - "plotters-backend", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802989b9fe1b674bc996ac7bed7b3012090a9b4cbfa0fe157ee3ea97e93e4ccd" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "proptest" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fcdab19deb5195a31cf7726a210015ff1496ba1464fd42cb4f537b8b01b471f" -dependencies = [ - "bit-set", - "bit-vec", - "bitflags 2.9.2", - "lazy_static", - "num-traits", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax", - "rusty-fork", - "tempfile", - "unarray", -] - -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quote" -version = "1.0.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" -dependencies = [ - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" -dependencies = [ - "getrandom 0.3.3", -] - -[[package]] -name = "rand_xorshift" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" -dependencies = [ - "rand_core", -] - -[[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "rdma-engine" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "bincode", - "bytes", - "chrono", - "clap", - "config", - "criterion", - "libc", - "libloading", - "memmap2", - "nix", - "parking_lot", - "proptest", - "rmp-serde", - "serde", - "tempfile", - "thiserror 1.0.69", - "tokio", - "tokio-util", - "tracing", - "tracing-subscriber", - "uuid", -] - -[[package]] -name = "redox_syscall" -version = "0.5.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" -dependencies = [ - "bitflags 2.9.2", -] - -[[package]] -name = "regex" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - -[[package]] -name = "rmp" -version = "0.8.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "228ed7c16fa39782c3b3468e974aec2795e9089153cd08ee2e9aefb3613334c4" -dependencies = [ - "byteorder", - "num-traits", - "paste", -] - -[[package]] -name = "rmp-serde" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e599a477cf9840e92f2cde9a7189e67b42c57532749bf90aea6ec10facd4db" -dependencies = [ - "byteorder", - "rmp", - "serde", -] - -[[package]] -name = "ron" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88073939a61e5b7680558e6be56b419e208420c2adb92be54921fa6b72283f1a" -dependencies = [ - "base64", - "bitflags 1.3.2", - "serde", -] - -[[package]] -name = "rust-ini" -version = "0.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6d5f2436026b4f6e79dc829837d467cc7e9a55ee40e750d716713540715a2df" -dependencies = [ - "cfg-if", - "ordered-multimap", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - -[[package]] -name = "rustix" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" -dependencies = [ - "bitflags 2.9.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.60.2", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "rusty-fork" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" -dependencies = [ - "fnv", - "quick-error", - "tempfile", - "wait-timeout", -] - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "serde" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.219" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.142" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tempfile" -version = "3.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" -dependencies = [ - "fastrand", - "getrandom 0.3.3", - "once_cell", - "rustix", - "windows-sys 0.59.0", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl 1.0.69", -] - -[[package]] -name = "thiserror" -version = "2.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80d76d3f064b981389ecb4b6b7f45a0bf9fdac1d5b9204c7bd6714fecc302850" -dependencies = [ - "thiserror-impl 2.0.15", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d29feb33e986b6ea906bd9c3559a856983f92371b3eaa5e83782a351623de0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "thread_local" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "tinytemplate" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "socket2", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-util" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" -dependencies = [ - "matchers", - "nu-ansi-term", - "once_cell", - "regex-automata", - "sharded-slab", - "smallvec", - "thread_local", - "tracing", - "tracing-core", - "tracing-log", -] - -[[package]] -name = "typenum" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" - -[[package]] -name = "ucd-trie" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" - -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - -[[package]] -name = "unicode-ident" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "uuid" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" -dependencies = [ - "getrandom 0.3.3", - "js-sys", - "serde", - "wasm-bindgen", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "wait-timeout" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" -dependencies = [ - "libc", -] - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "winapi-util" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" -dependencies = [ - "windows-sys 0.59.0", -] - -[[package]] -name = "windows-core" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" -dependencies = [ - "windows-implement", - "windows-interface", - "windows-link", - "windows-result", - "windows-strings", -] - -[[package]] -name = "windows-implement" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-interface" -version = "0.59.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-result" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-strings" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.3", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" - -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.9.2", -] - -[[package]] -name = "yaml-rust" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "zerocopy" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml b/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml deleted file mode 100644 index 5b355fffc..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/Cargo.toml +++ /dev/null @@ -1,74 +0,0 @@ -[package] -name = "rdma-engine" -version = "0.1.0" -edition = "2021" -authors = ["SeaweedFS Team "] -description = "High-performance RDMA engine for SeaweedFS sidecar" -license = "Apache-2.0" - -[[bin]] -name = "rdma-engine-server" -path = "src/main.rs" - -[lib] -name = "rdma_engine" -path = "src/lib.rs" - -[dependencies] -# UCX (Unified Communication X) for high-performance networking -# Much better than direct libibverbs - provides unified API across transports -libc = "0.2" -libloading = "0.8" # Dynamic loading of UCX libraries - -# Async runtime and networking -tokio = { version = "1.0", features = ["full"] } -tokio-util = "0.7" - -# Serialization for IPC -serde = { version = "1.0", features = ["derive"] } -bincode = "1.3" -rmp-serde = "1.1" # MessagePack for efficient IPC - -# Error handling and logging -anyhow = "1.0" -thiserror = "1.0" -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } - -# UUID and time handling -uuid = { version = "1.0", features = ["v4", "serde"] } -chrono = { version = "0.4", features = ["serde"] } - -# Memory management and utilities -memmap2 = "0.9" -bytes = "1.11" -parking_lot = "0.12" # Fast mutexes - -# IPC and networking -nix = { version = "0.27", features = ["mman"] } # Unix domain sockets and system calls -async-trait = "0.1" # Async traits - -# Configuration -clap = { version = "4.0", features = ["derive"] } -config = "0.13" - -[dev-dependencies] -proptest = "1.0" -criterion = "0.5" -tempfile = "3.0" - -[features] -default = ["mock-ucx"] -mock-ucx = [] -real-ucx = [] # UCX integration for production RDMA - -[profile.release] -opt-level = 3 -lto = true -codegen-units = 1 -panic = "abort" - - - -[package.metadata.docs.rs] -features = ["real-rdma"] diff --git a/seaweedfs-rdma-sidecar/rdma-engine/README.md b/seaweedfs-rdma-sidecar/rdma-engine/README.md deleted file mode 100644 index 1c7d575ae..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/README.md +++ /dev/null @@ -1,88 +0,0 @@ -# UCX-based RDMA Engine for SeaweedFS - -High-performance Rust-based communication engine for SeaweedFS using [UCX (Unified Communication X)](https://github.com/openucx/ucx) framework that provides optimized data transfers across multiple transports including RDMA (InfiniBand/RoCE), TCP, and shared memory. - -## ๐Ÿš€ **Complete Rust RDMA Sidecar Scaffolded!** - -I've successfully created a comprehensive Rust RDMA engine with the following components: - -### โœ… **What's Implemented** - -1. **Complete Project Structure**: - - `src/lib.rs` - Main library with engine management - - `src/main.rs` - Binary entry point with CLI - - `src/error.rs` - Comprehensive error types - - `src/rdma.rs` - RDMA operations (mock & real) - - `src/ipc.rs` - IPC communication with Go sidecar - - `src/session.rs` - Session management - - `src/memory.rs` - Memory management and pooling - -2. **Advanced Features**: - - Mock RDMA implementation for development - - Real RDMA stubs ready for `libibverbs` integration - - High-performance memory management with pooling - - HugePage support for large allocations - - Thread-safe session management with expiration - - MessagePack-based IPC protocol - - Comprehensive error handling and recovery - - Performance monitoring and statistics - -3. **Production-Ready Architecture**: - - Async/await throughout for high concurrency - - Zero-copy memory operations where possible - - Proper resource cleanup and garbage collection - - Signal handling for graceful shutdown - - Configurable via CLI flags and config files - - Extensive logging and metrics - -### ๐Ÿ› ๏ธ **Current Status** - -The scaffolding is **functionally complete** but has some compilation errors that need to be resolved: - -1. **Async Trait Object Issues** - Rust doesn't support async methods in trait objects -2. **Stream Ownership** - BufReader/BufWriter ownership needs fixing -3. **Memory Management** - Some lifetime and cloning issues - -### ๐Ÿ”ง **Next Steps to Complete** - -1. **Fix Compilation Errors** (1-2 hours): - - Replace trait objects with enums for RDMA context - - Fix async trait issues with concrete types - - Resolve memory ownership issues - -2. **Integration with Go Sidecar** (2-4 hours): - - Update Go sidecar to communicate with Rust engine - - Implement Unix domain socket protocol - - Add fallback when Rust engine is unavailable - -3. **RDMA Hardware Integration** (1-2 weeks): - - Add `libibverbs` FFI bindings - - Implement real RDMA operations - - Test on actual InfiniBand hardware - -### ๐Ÿ“Š **Architecture Overview** - -``` -โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” IPC โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -โ”‚ Go Control Plane โ”‚โ—„โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บโ”‚ Rust Data Plane โ”‚ -โ”‚ โ”‚ ~300ns โ”‚ โ”‚ -โ”‚ โ€ข gRPC Server โ”‚ โ”‚ โ€ข RDMA Operations โ”‚ -โ”‚ โ€ข Session Mgmt โ”‚ โ”‚ โ€ข Memory Mgmt โ”‚ -โ”‚ โ€ข HTTP Fallback โ”‚ โ”‚ โ€ข Hardware Access โ”‚ -โ”‚ โ€ข Error Handling โ”‚ โ”‚ โ€ข Zero-Copy I/O โ”‚ -โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -``` - -### ๐ŸŽฏ **Performance Expectations** - -- **Mock RDMA**: ~150ns per operation (current) -- **Real RDMA**: ~50ns per operation (projected) -- **Memory Operations**: Zero-copy with hugepage support -- **Session Throughput**: 1M+ sessions/second -- **IPC Overhead**: ~300ns (Unix domain sockets) - -## ๐Ÿš€ **Ready for Hardware Integration** - -This Rust RDMA engine provides a **solid foundation** for high-performance RDMA acceleration. The architecture is sound, the error handling is comprehensive, and the memory management is optimized for RDMA workloads. - -**Next milestone**: Fix compilation errors and integrate with the existing Go sidecar for end-to-end testing! ๐ŸŽฏ diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/error.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/error.rs deleted file mode 100644 index be60ef4aa..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/error.rs +++ /dev/null @@ -1,269 +0,0 @@ -//! Error types and handling for the RDMA engine - -// use std::fmt; // Unused for now -use thiserror::Error; - -/// Result type alias for RDMA operations -pub type RdmaResult = Result; - -/// Comprehensive error types for RDMA operations -#[derive(Error, Debug)] -pub enum RdmaError { - /// RDMA device not found or unavailable - #[error("RDMA device '{device}' not found or unavailable")] - DeviceNotFound { device: String }, - - /// Failed to initialize RDMA context - #[error("Failed to initialize RDMA context: {reason}")] - ContextInitFailed { reason: String }, - - /// Failed to allocate protection domain - #[error("Failed to allocate protection domain: {reason}")] - PdAllocFailed { reason: String }, - - /// Failed to create completion queue - #[error("Failed to create completion queue: {reason}")] - CqCreationFailed { reason: String }, - - /// Failed to create queue pair - #[error("Failed to create queue pair: {reason}")] - QpCreationFailed { reason: String }, - - /// Memory registration failed - #[error("Memory registration failed: {reason}")] - MemoryRegFailed { reason: String }, - - /// RDMA operation failed - #[error("RDMA operation failed: {operation}, status: {status}")] - OperationFailed { operation: String, status: i32 }, - - /// Session not found - #[error("Session '{session_id}' not found")] - SessionNotFound { session_id: String }, - - /// Session expired - #[error("Session '{session_id}' has expired")] - SessionExpired { session_id: String }, - - /// Too many active sessions - #[error("Maximum number of sessions ({max_sessions}) exceeded")] - TooManySessions { max_sessions: usize }, - - /// IPC communication error - #[error("IPC communication error: {reason}")] - IpcError { reason: String }, - - /// Serialization/deserialization error - #[error("Serialization error: {reason}")] - SerializationError { reason: String }, - - /// Invalid request parameters - #[error("Invalid request: {reason}")] - InvalidRequest { reason: String }, - - /// Insufficient buffer space - #[error("Insufficient buffer space: requested {requested}, available {available}")] - InsufficientBuffer { requested: usize, available: usize }, - - /// Hardware not supported - #[error("Hardware not supported: {reason}")] - UnsupportedHardware { reason: String }, - - /// System resource exhausted - #[error("System resource exhausted: {resource}")] - ResourceExhausted { resource: String }, - - /// Permission denied - #[error("Permission denied: {operation}")] - PermissionDenied { operation: String }, - - /// Network timeout - #[error("Network timeout after {timeout_ms}ms")] - NetworkTimeout { timeout_ms: u64 }, - - /// I/O error - #[error("I/O error: {0}")] - Io(#[from] std::io::Error), - - /// Generic error for unexpected conditions - #[error("Internal error: {reason}")] - Internal { reason: String }, -} - -impl RdmaError { - /// Create a new DeviceNotFound error - pub fn device_not_found(device: impl Into) -> Self { - Self::DeviceNotFound { device: device.into() } - } - - /// Create a new ContextInitFailed error - pub fn context_init_failed(reason: impl Into) -> Self { - Self::ContextInitFailed { reason: reason.into() } - } - - /// Create a new MemoryRegFailed error - pub fn memory_reg_failed(reason: impl Into) -> Self { - Self::MemoryRegFailed { reason: reason.into() } - } - - /// Create a new OperationFailed error - pub fn operation_failed(operation: impl Into, status: i32) -> Self { - Self::OperationFailed { - operation: operation.into(), - status - } - } - - /// Create a new SessionNotFound error - pub fn session_not_found(session_id: impl Into) -> Self { - Self::SessionNotFound { session_id: session_id.into() } - } - - /// Create a new IpcError - pub fn ipc_error(reason: impl Into) -> Self { - Self::IpcError { reason: reason.into() } - } - - /// Create a new InvalidRequest error - pub fn invalid_request(reason: impl Into) -> Self { - Self::InvalidRequest { reason: reason.into() } - } - - /// Create a new Internal error - pub fn internal(reason: impl Into) -> Self { - Self::Internal { reason: reason.into() } - } - - /// Check if this error is recoverable - pub fn is_recoverable(&self) -> bool { - match self { - // Network and temporary errors are recoverable - Self::NetworkTimeout { .. } | - Self::ResourceExhausted { .. } | - Self::TooManySessions { .. } | - Self::InsufficientBuffer { .. } => true, - - // Session errors are recoverable (can retry with new session) - Self::SessionNotFound { .. } | - Self::SessionExpired { .. } => true, - - // Hardware and system errors are generally not recoverable - Self::DeviceNotFound { .. } | - Self::ContextInitFailed { .. } | - Self::UnsupportedHardware { .. } | - Self::PermissionDenied { .. } => false, - - // IPC errors might be recoverable - Self::IpcError { .. } | - Self::SerializationError { .. } => true, - - // Invalid requests are not recoverable without fixing the request - Self::InvalidRequest { .. } => false, - - // RDMA operation failures might be recoverable - Self::OperationFailed { .. } => true, - - // Memory and resource allocation failures depend on the cause - Self::PdAllocFailed { .. } | - Self::CqCreationFailed { .. } | - Self::QpCreationFailed { .. } | - Self::MemoryRegFailed { .. } => false, - - // I/O errors might be recoverable - Self::Io(_) => true, - - // Internal errors are generally not recoverable - Self::Internal { .. } => false, - } - } - - /// Get error category for metrics and logging - pub fn category(&self) -> &'static str { - match self { - Self::DeviceNotFound { .. } | - Self::ContextInitFailed { .. } | - Self::UnsupportedHardware { .. } => "hardware", - - Self::PdAllocFailed { .. } | - Self::CqCreationFailed { .. } | - Self::QpCreationFailed { .. } | - Self::MemoryRegFailed { .. } => "resource", - - Self::OperationFailed { .. } => "rdma", - - Self::SessionNotFound { .. } | - Self::SessionExpired { .. } | - Self::TooManySessions { .. } => "session", - - Self::IpcError { .. } | - Self::SerializationError { .. } => "ipc", - - Self::InvalidRequest { .. } => "request", - - Self::InsufficientBuffer { .. } | - Self::ResourceExhausted { .. } => "capacity", - - Self::PermissionDenied { .. } => "security", - - Self::NetworkTimeout { .. } => "network", - - Self::Io(_) => "io", - - Self::Internal { .. } => "internal", - } - } -} - -/// Convert from various RDMA library error codes -impl From for RdmaError { - fn from(errno: i32) -> Self { - match errno { - libc::ENODEV => Self::DeviceNotFound { - device: "unknown".to_string() - }, - libc::ENOMEM => Self::ResourceExhausted { - resource: "memory".to_string() - }, - libc::EPERM | libc::EACCES => Self::PermissionDenied { - operation: "RDMA operation".to_string() - }, - libc::ETIMEDOUT => Self::NetworkTimeout { - timeout_ms: 5000 - }, - libc::ENOSPC => Self::InsufficientBuffer { - requested: 0, - available: 0 - }, - _ => Self::Internal { - reason: format!("System error: {}", errno) - }, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_error_creation() { - let err = RdmaError::device_not_found("mlx5_0"); - assert!(matches!(err, RdmaError::DeviceNotFound { .. })); - assert_eq!(err.category(), "hardware"); - assert!(!err.is_recoverable()); - } - - #[test] - fn test_error_recoverability() { - assert!(RdmaError::NetworkTimeout { timeout_ms: 1000 }.is_recoverable()); - assert!(!RdmaError::DeviceNotFound { device: "test".to_string() }.is_recoverable()); - assert!(RdmaError::SessionExpired { session_id: "test".to_string() }.is_recoverable()); - } - - #[test] - fn test_error_display() { - let err = RdmaError::InvalidRequest { reason: "missing field".to_string() }; - assert!(err.to_string().contains("Invalid request")); - assert!(err.to_string().contains("missing field")); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/ipc.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/ipc.rs deleted file mode 100644 index a578c2d7d..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/ipc.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! IPC (Inter-Process Communication) module for communicating with Go sidecar -//! -//! This module handles high-performance IPC between the Rust RDMA engine and -//! the Go control plane sidecar using Unix domain sockets and MessagePack serialization. - -use crate::{RdmaError, RdmaResult, rdma::RdmaContext, session::SessionManager}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use std::sync::atomic::{AtomicU64, Ordering}; -use tokio::net::{UnixListener, UnixStream}; -use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader, BufWriter}; -use tracing::{info, debug, error}; -use uuid::Uuid; -use std::path::Path; - -/// Atomic counter for generating unique work request IDs -/// This ensures no hash collisions that could cause incorrect completion handling -static NEXT_WR_ID: AtomicU64 = AtomicU64::new(1); - -/// IPC message types between Go sidecar and Rust RDMA engine -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", content = "data")] -pub enum IpcMessage { - /// Request to start an RDMA read operation - StartRead(StartReadRequest), - /// Response with RDMA session information - StartReadResponse(StartReadResponse), - - /// Request to complete an RDMA operation - CompleteRead(CompleteReadRequest), - /// Response confirming completion - CompleteReadResponse(CompleteReadResponse), - - /// Request for engine capabilities - GetCapabilities(GetCapabilitiesRequest), - /// Response with engine capabilities - GetCapabilitiesResponse(GetCapabilitiesResponse), - - /// Health check ping - Ping(PingRequest), - /// Ping response - Pong(PongResponse), - - /// Error response - Error(ErrorResponse), -} - -/// Request to start RDMA read operation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StartReadRequest { - /// Volume ID in SeaweedFS - pub volume_id: u32, - /// Needle ID in SeaweedFS - pub needle_id: u64, - /// Needle cookie for validation - pub cookie: u32, - /// File offset within the needle data - pub offset: u64, - /// Size to read (0 = entire needle) - pub size: u64, - /// Remote memory address from Go sidecar - pub remote_addr: u64, - /// Remote key for RDMA access - pub remote_key: u32, - /// Session timeout in seconds - pub timeout_secs: u64, - /// Authentication token (optional) - pub auth_token: Option, -} - -/// Response with RDMA session details -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StartReadResponse { - /// Unique session identifier - pub session_id: String, - /// Local buffer address for RDMA - pub local_addr: u64, - /// Local key for RDMA operations - pub local_key: u32, - /// Actual size that will be transferred - pub transfer_size: u64, - /// Expected CRC checksum - pub expected_crc: u32, - /// Session expiration timestamp (Unix nanoseconds) - pub expires_at_ns: u64, -} - -/// Request to complete RDMA operation -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CompleteReadRequest { - /// Session ID to complete - pub session_id: String, - /// Whether the operation was successful - pub success: bool, - /// Actual bytes transferred - pub bytes_transferred: u64, - /// Client-computed CRC (for verification) - pub client_crc: Option, - /// Error message if failed - pub error_message: Option, -} - -/// Response confirming completion -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CompleteReadResponse { - /// Whether completion was successful - pub success: bool, - /// Server-computed CRC for verification - pub server_crc: Option, - /// Any cleanup messages - pub message: Option, -} - -/// Request for engine capabilities -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GetCapabilitiesRequest { - /// Client identifier - pub client_id: Option, -} - -/// Response with engine capabilities -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct GetCapabilitiesResponse { - /// RDMA device name - pub device_name: String, - /// RDMA device vendor ID - pub vendor_id: u32, - /// Maximum transfer size in bytes - pub max_transfer_size: u64, - /// Maximum concurrent sessions - pub max_sessions: usize, - /// Current active sessions - pub active_sessions: usize, - /// Device port GID - pub port_gid: String, - /// Device port LID - pub port_lid: u16, - /// Supported authentication methods - pub supported_auth: Vec, - /// Engine version - pub version: String, - /// Whether real RDMA hardware is available - pub real_rdma: bool, -} - -/// Health check ping request -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PingRequest { - /// Client timestamp (Unix nanoseconds) - pub timestamp_ns: u64, - /// Client identifier - pub client_id: Option, -} - -/// Ping response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PongResponse { - /// Original client timestamp - pub client_timestamp_ns: u64, - /// Server timestamp (Unix nanoseconds) - pub server_timestamp_ns: u64, - /// Round-trip time in nanoseconds (server perspective) - pub server_rtt_ns: u64, -} - -/// Error response -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ErrorResponse { - /// Error code - pub code: String, - /// Human-readable error message - pub message: String, - /// Error category - pub category: String, - /// Whether the error is recoverable - pub recoverable: bool, -} - -impl From<&RdmaError> for ErrorResponse { - fn from(error: &RdmaError) -> Self { - Self { - code: format!("{:?}", error), - message: error.to_string(), - category: error.category().to_string(), - recoverable: error.is_recoverable(), - } - } -} - -/// IPC server handling communication with Go sidecar -pub struct IpcServer { - socket_path: String, - listener: Option, - rdma_context: Arc, - session_manager: Arc, - shutdown_flag: Arc>, -} - -impl IpcServer { - /// Create new IPC server - pub async fn new( - socket_path: &str, - rdma_context: Arc, - session_manager: Arc, - ) -> RdmaResult { - // Remove existing socket if it exists - if Path::new(socket_path).exists() { - std::fs::remove_file(socket_path) - .map_err(|e| RdmaError::ipc_error(format!("Failed to remove existing socket: {}", e)))?; - } - - Ok(Self { - socket_path: socket_path.to_string(), - listener: None, - rdma_context, - session_manager, - shutdown_flag: Arc::new(parking_lot::RwLock::new(false)), - }) - } - - /// Start the IPC server - pub async fn run(&mut self) -> RdmaResult<()> { - let listener = UnixListener::bind(&self.socket_path) - .map_err(|e| RdmaError::ipc_error(format!("Failed to bind Unix socket: {}", e)))?; - - info!("๐ŸŽฏ IPC server listening on: {}", self.socket_path); - self.listener = Some(listener); - - if let Some(ref listener) = self.listener { - loop { - // Check shutdown flag - if *self.shutdown_flag.read() { - info!("IPC server shutting down"); - break; - } - - // Accept connection with timeout - let accept_result = tokio::time::timeout( - tokio::time::Duration::from_millis(100), - listener.accept() - ).await; - - match accept_result { - Ok(Ok((stream, addr))) => { - debug!("New IPC connection from: {:?}", addr); - - // Spawn handler for this connection - let rdma_context = self.rdma_context.clone(); - let session_manager = self.session_manager.clone(); - let shutdown_flag = self.shutdown_flag.clone(); - - tokio::spawn(async move { - if let Err(e) = Self::handle_connection(stream, rdma_context, session_manager, shutdown_flag).await { - error!("IPC connection error: {}", e); - } - }); - } - Ok(Err(e)) => { - error!("Failed to accept IPC connection: {}", e); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - Err(_) => { - // Timeout - continue loop to check shutdown flag - continue; - } - } - } - } - - Ok(()) - } - - /// Handle a single IPC connection - async fn handle_connection( - stream: UnixStream, - rdma_context: Arc, - session_manager: Arc, - shutdown_flag: Arc>, - ) -> RdmaResult<()> { - let (reader_half, writer_half) = stream.into_split(); - let mut reader = BufReader::new(reader_half); - let mut writer = BufWriter::new(writer_half); - - let mut buffer = Vec::with_capacity(4096); - - loop { - // Check shutdown - if *shutdown_flag.read() { - break; - } - - // Read message length (4 bytes) - let mut len_bytes = [0u8; 4]; - match tokio::time::timeout( - tokio::time::Duration::from_millis(100), - reader.read_exact(&mut len_bytes) - ).await { - Ok(Ok(_)) => {}, - Ok(Err(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => { - debug!("IPC connection closed by peer"); - break; - } - Ok(Err(e)) => return Err(RdmaError::ipc_error(format!("Read error: {}", e))), - Err(_) => continue, // Timeout, check shutdown flag - } - - let msg_len = u32::from_le_bytes(len_bytes) as usize; - if msg_len > 1024 * 1024 { // 1MB max message size - return Err(RdmaError::ipc_error("Message too large")); - } - - // Read message data - buffer.clear(); - buffer.resize(msg_len, 0); - reader.read_exact(&mut buffer).await - .map_err(|e| RdmaError::ipc_error(format!("Failed to read message: {}", e)))?; - - // Deserialize message - let request: IpcMessage = rmp_serde::from_slice(&buffer) - .map_err(|e| RdmaError::SerializationError { reason: e.to_string() })?; - - debug!("Received IPC message: {:?}", request); - - // Process message - let response = Self::process_message( - request, - &rdma_context, - &session_manager, - ).await; - - // Serialize response - let response_data = rmp_serde::to_vec(&response) - .map_err(|e| RdmaError::SerializationError { reason: e.to_string() })?; - - // Send response - let response_len = (response_data.len() as u32).to_le_bytes(); - writer.write_all(&response_len).await - .map_err(|e| RdmaError::ipc_error(format!("Failed to write response length: {}", e)))?; - writer.write_all(&response_data).await - .map_err(|e| RdmaError::ipc_error(format!("Failed to write response: {}", e)))?; - writer.flush().await - .map_err(|e| RdmaError::ipc_error(format!("Failed to flush response: {}", e)))?; - - debug!("Sent IPC response"); - } - - Ok(()) - } - - /// Process IPC message and generate response - async fn process_message( - message: IpcMessage, - rdma_context: &Arc, - session_manager: &Arc, - ) -> IpcMessage { - match message { - IpcMessage::Ping(req) => { - let server_timestamp = chrono::Utc::now().timestamp_nanos_opt().unwrap_or(0) as u64; - IpcMessage::Pong(PongResponse { - client_timestamp_ns: req.timestamp_ns, - server_timestamp_ns: server_timestamp, - server_rtt_ns: server_timestamp.saturating_sub(req.timestamp_ns), - }) - } - - IpcMessage::GetCapabilities(_req) => { - let device_info = rdma_context.device_info(); - let active_sessions = session_manager.active_session_count().await; - - IpcMessage::GetCapabilitiesResponse(GetCapabilitiesResponse { - device_name: device_info.name.clone(), - vendor_id: device_info.vendor_id, - max_transfer_size: device_info.max_mr_size, - max_sessions: session_manager.max_sessions(), - active_sessions, - port_gid: device_info.port_gid.clone(), - port_lid: device_info.port_lid, - supported_auth: vec!["none".to_string()], - version: env!("CARGO_PKG_VERSION").to_string(), - real_rdma: cfg!(feature = "real-ucx"), - }) - } - - IpcMessage::StartRead(req) => { - match Self::handle_start_read(req, rdma_context, session_manager).await { - Ok(response) => IpcMessage::StartReadResponse(response), - Err(error) => IpcMessage::Error(ErrorResponse::from(&error)), - } - } - - IpcMessage::CompleteRead(req) => { - match Self::handle_complete_read(req, session_manager).await { - Ok(response) => IpcMessage::CompleteReadResponse(response), - Err(error) => IpcMessage::Error(ErrorResponse::from(&error)), - } - } - - _ => IpcMessage::Error(ErrorResponse { - code: "UNSUPPORTED_MESSAGE".to_string(), - message: "Unsupported message type".to_string(), - category: "request".to_string(), - recoverable: true, - }), - } - } - - /// Handle StartRead request - async fn handle_start_read( - req: StartReadRequest, - rdma_context: &Arc, - session_manager: &Arc, - ) -> RdmaResult { - info!("๐Ÿš€ Starting RDMA read: volume={}, needle={}, size={}", - req.volume_id, req.needle_id, req.size); - - // Create session - let session_id = Uuid::new_v4().to_string(); - let transfer_size = if req.size == 0 { 65536 } else { req.size }; // Default 64KB - - // Allocate local buffer - let buffer = vec![0u8; transfer_size as usize]; - let local_addr = buffer.as_ptr() as u64; - - // Register memory for RDMA - let memory_region = rdma_context.register_memory(local_addr, transfer_size as usize).await?; - - // Create and store session - session_manager.create_session( - session_id.clone(), - req.volume_id, - req.needle_id, - req.remote_addr, - req.remote_key, - transfer_size, - buffer, - memory_region.clone(), - chrono::Duration::seconds(req.timeout_secs as i64), - ).await?; - - // Perform RDMA read with unique work request ID - // Use atomic counter to avoid hash collisions that could cause incorrect completion handling - let wr_id = NEXT_WR_ID.fetch_add(1, Ordering::Relaxed); - rdma_context.post_read( - local_addr, - req.remote_addr, - req.remote_key, - transfer_size as usize, - wr_id, - ).await?; - - // Poll for completion - let completions = rdma_context.poll_completion(1).await?; - if completions.is_empty() { - return Err(RdmaError::operation_failed("RDMA read", -1)); - } - - let completion = &completions[0]; - if completion.status != crate::rdma::CompletionStatus::Success { - return Err(RdmaError::operation_failed("RDMA read", completion.status as i32)); - } - - info!("โœ… RDMA read completed: {} bytes", completion.byte_len); - - let expires_at = chrono::Utc::now() + chrono::Duration::seconds(req.timeout_secs as i64); - - Ok(StartReadResponse { - session_id, - local_addr, - local_key: memory_region.lkey, - transfer_size, - expected_crc: 0x12345678, // Mock CRC - expires_at_ns: expires_at.timestamp_nanos_opt().unwrap_or(0) as u64, - }) - } - - /// Handle CompleteRead request - async fn handle_complete_read( - req: CompleteReadRequest, - session_manager: &Arc, - ) -> RdmaResult { - info!("๐Ÿ Completing RDMA read session: {}", req.session_id); - - // Clean up session - session_manager.remove_session(&req.session_id).await?; - - Ok(CompleteReadResponse { - success: req.success, - server_crc: Some(0x12345678), // Mock CRC - message: Some("Session completed successfully".to_string()), - }) - } - - /// Shutdown the IPC server - pub async fn shutdown(&mut self) -> RdmaResult<()> { - info!("Shutting down IPC server"); - *self.shutdown_flag.write() = true; - - // Remove socket file - if Path::new(&self.socket_path).exists() { - std::fs::remove_file(&self.socket_path) - .map_err(|e| RdmaError::ipc_error(format!("Failed to remove socket file: {}", e)))?; - } - - Ok(()) - } -} - - - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_error_response_conversion() { - let error = RdmaError::device_not_found("mlx5_0"); - let response = ErrorResponse::from(&error); - - assert!(response.message.contains("mlx5_0")); - assert_eq!(response.category, "hardware"); - assert!(!response.recoverable); - } - - #[test] - fn test_message_serialization() { - let request = IpcMessage::Ping(PingRequest { - timestamp_ns: 12345, - client_id: Some("test".to_string()), - }); - - let serialized = rmp_serde::to_vec(&request).unwrap(); - let deserialized: IpcMessage = rmp_serde::from_slice(&serialized).unwrap(); - - match deserialized { - IpcMessage::Ping(ping) => { - assert_eq!(ping.timestamp_ns, 12345); - assert_eq!(ping.client_id, Some("test".to_string())); - } - _ => panic!("Wrong message type"), - } - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs deleted file mode 100644 index c92dcf91a..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/lib.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! High-Performance RDMA Engine for SeaweedFS -//! -//! This crate provides a high-performance RDMA (Remote Direct Memory Access) engine -//! designed to accelerate data transfer operations in SeaweedFS. It communicates with -//! the Go-based sidecar via IPC and handles the performance-critical RDMA operations. -//! -//! # Architecture -//! -//! ```text -//! โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” IPC โ”Œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ” -//! โ”‚ Go Control Plane โ”‚โ—„โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ–บโ”‚ Rust Data Plane โ”‚ -//! โ”‚ โ”‚ ~300ns โ”‚ โ”‚ -//! โ”‚ โ€ข gRPC Server โ”‚ โ”‚ โ€ข RDMA Operations โ”‚ -//! โ”‚ โ€ข Session Mgmt โ”‚ โ”‚ โ€ข Memory Mgmt โ”‚ -//! โ”‚ โ€ข HTTP Fallback โ”‚ โ”‚ โ€ข Hardware Access โ”‚ -//! โ”‚ โ€ข Error Handling โ”‚ โ”‚ โ€ข Zero-Copy I/O โ”‚ -//! โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ -//! ``` -//! -//! # Features -//! -//! - `mock-rdma` (default): Mock RDMA operations for testing and development -//! - `real-rdma`: Real RDMA hardware integration using rdma-core bindings - -use std::sync::Arc; -use anyhow::Result; - -pub mod ucx; -pub mod rdma; -pub mod ipc; -pub mod session; -pub mod memory; -pub mod error; - -pub use error::{RdmaError, RdmaResult}; - -/// Configuration for the RDMA engine -#[derive(Debug, Clone)] -pub struct RdmaEngineConfig { - /// RDMA device name (e.g., "mlx5_0") - pub device_name: String, - /// RDMA port number - pub port: u16, - /// Maximum number of concurrent sessions - pub max_sessions: usize, - /// Session timeout in seconds - pub session_timeout_secs: u64, - /// Memory buffer size in bytes - pub buffer_size: usize, - /// IPC socket path - pub ipc_socket_path: String, - /// Enable debug logging - pub debug: bool, -} - -impl Default for RdmaEngineConfig { - fn default() -> Self { - Self { - device_name: "mlx5_0".to_string(), - port: 18515, - max_sessions: 1000, - session_timeout_secs: 300, // 5 minutes - buffer_size: 1024 * 1024 * 1024, // 1GB - ipc_socket_path: "/tmp/rdma-engine.sock".to_string(), - debug: false, - } - } -} - -/// Main RDMA engine instance -pub struct RdmaEngine { - config: RdmaEngineConfig, - rdma_context: Arc, - session_manager: Arc, - ipc_server: Option, -} - -impl RdmaEngine { - /// Create a new RDMA engine with the given configuration - pub async fn new(config: RdmaEngineConfig) -> Result { - tracing::info!("Initializing RDMA engine with config: {:?}", config); - - // Initialize RDMA context - let rdma_context = Arc::new(rdma::RdmaContext::new(&config).await?); - - // Initialize session manager - let session_manager = Arc::new(session::SessionManager::new( - config.max_sessions, - std::time::Duration::from_secs(config.session_timeout_secs), - )); - - Ok(Self { - config, - rdma_context, - session_manager, - ipc_server: None, - }) - } - - /// Start the RDMA engine server - pub async fn run(&mut self) -> Result<()> { - tracing::info!("Starting RDMA engine server on {}", self.config.ipc_socket_path); - - // Start IPC server - let ipc_server = ipc::IpcServer::new( - &self.config.ipc_socket_path, - self.rdma_context.clone(), - self.session_manager.clone(), - ).await?; - - self.ipc_server = Some(ipc_server); - - // Start session cleanup task - let session_manager = self.session_manager.clone(); - tokio::spawn(async move { - session_manager.start_cleanup_task().await; - }); - - // Run IPC server - if let Some(ref mut server) = self.ipc_server { - server.run().await?; - } - - Ok(()) - } - - /// Shutdown the RDMA engine - pub async fn shutdown(&mut self) -> Result<()> { - tracing::info!("Shutting down RDMA engine"); - - if let Some(ref mut server) = self.ipc_server { - server.shutdown().await?; - } - - self.session_manager.shutdown().await; - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_rdma_engine_creation() { - let config = RdmaEngineConfig::default(); - let result = RdmaEngine::new(config).await; - - // Should succeed with mock RDMA - assert!(result.is_ok()); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/main.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/main.rs deleted file mode 100644 index 996d3a9d5..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/main.rs +++ /dev/null @@ -1,175 +0,0 @@ -//! RDMA Engine Server -//! -//! High-performance RDMA engine server that communicates with the Go sidecar -//! via IPC and handles RDMA operations with zero-copy semantics. -//! -//! Usage: -//! ```bash -//! rdma-engine-server --device mlx5_0 --port 18515 --ipc-socket /tmp/rdma-engine.sock -//! ``` - -use clap::Parser; -use rdma_engine::{RdmaEngine, RdmaEngineConfig}; -use std::path::PathBuf; -use tracing::{info, error}; -use tracing_subscriber::{EnvFilter, fmt::layer, prelude::*}; - -#[derive(Parser)] -#[command( - name = "rdma-engine-server", - about = "High-performance RDMA engine for SeaweedFS", - version = env!("CARGO_PKG_VERSION") -)] -struct Args { - /// UCX device name preference (e.g., mlx5_0, or 'auto' for UCX auto-selection) - #[arg(short, long, default_value = "auto")] - device: String, - - /// RDMA port number - #[arg(short, long, default_value_t = 18515)] - port: u16, - - /// Maximum number of concurrent sessions - #[arg(long, default_value_t = 1000)] - max_sessions: usize, - - /// Session timeout in seconds - #[arg(long, default_value_t = 300)] - session_timeout: u64, - - /// Memory buffer size in bytes - #[arg(long, default_value_t = 1024 * 1024 * 1024)] - buffer_size: usize, - - /// IPC socket path - #[arg(long, default_value = "/tmp/rdma-engine.sock")] - ipc_socket: PathBuf, - - /// Enable debug logging - #[arg(long)] - debug: bool, - - /// Configuration file path - #[arg(short, long)] - config: Option, -} - -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let args = Args::parse(); - - // Initialize tracing - let filter = if args.debug { - EnvFilter::try_from_default_env() - .or_else(|_| EnvFilter::try_new("debug")) - .unwrap() - } else { - EnvFilter::try_from_default_env() - .or_else(|_| EnvFilter::try_new("info")) - .unwrap() - }; - - tracing_subscriber::registry() - .with(layer().with_target(false)) - .with(filter) - .init(); - - info!("๐Ÿš€ Starting SeaweedFS UCX RDMA Engine Server"); - info!(" Version: {}", env!("CARGO_PKG_VERSION")); - info!(" UCX Device Preference: {}", args.device); - info!(" Port: {}", args.port); - info!(" Max Sessions: {}", args.max_sessions); - info!(" Buffer Size: {} bytes", args.buffer_size); - info!(" IPC Socket: {}", args.ipc_socket.display()); - info!(" Debug Mode: {}", args.debug); - - // Load configuration - let config = RdmaEngineConfig { - device_name: args.device, - port: args.port, - max_sessions: args.max_sessions, - session_timeout_secs: args.session_timeout, - buffer_size: args.buffer_size, - ipc_socket_path: args.ipc_socket.to_string_lossy().to_string(), - debug: args.debug, - }; - - // Override with config file if provided - if let Some(config_path) = args.config { - info!("Loading configuration from: {}", config_path.display()); - // TODO: Implement configuration file loading - } - - // Create and run RDMA engine - let mut engine = match RdmaEngine::new(config).await { - Ok(engine) => { - info!("โœ… RDMA engine initialized successfully"); - engine - } - Err(e) => { - error!("โŒ Failed to initialize RDMA engine: {}", e); - return Err(e); - } - }; - - // Set up signal handlers for graceful shutdown - let mut sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?; - let mut sigint = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())?; - - // Run engine in background - let engine_handle = tokio::spawn(async move { - if let Err(e) = engine.run().await { - error!("RDMA engine error: {}", e); - return Err(e); - } - Ok(()) - }); - - info!("๐ŸŽฏ RDMA engine is running and ready to accept connections"); - info!(" Send SIGTERM or SIGINT to shutdown gracefully"); - - // Wait for shutdown signal - tokio::select! { - _ = sigterm.recv() => { - info!("๐Ÿ“ก Received SIGTERM, shutting down gracefully"); - } - _ = sigint.recv() => { - info!("๐Ÿ“ก Received SIGINT (Ctrl+C), shutting down gracefully"); - } - result = engine_handle => { - match result { - Ok(Ok(())) => info!("๐Ÿ RDMA engine completed successfully"), - Ok(Err(e)) => { - error!("โŒ RDMA engine failed: {}", e); - return Err(e); - } - Err(e) => { - error!("โŒ RDMA engine task panicked: {}", e); - return Err(anyhow::anyhow!("Engine task panicked: {}", e)); - } - } - } - } - - info!("๐Ÿ›‘ RDMA engine server shut down complete"); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_args_parsing() { - let args = Args::try_parse_from(&[ - "rdma-engine-server", - "--device", "mlx5_0", - "--port", "18515", - "--debug" - ]).unwrap(); - - assert_eq!(args.device, "mlx5_0"); - assert_eq!(args.port, 18515); - assert!(args.debug); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/memory.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/memory.rs deleted file mode 100644 index 17a9a5b1d..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/memory.rs +++ /dev/null @@ -1,630 +0,0 @@ -//! Memory management for RDMA operations -//! -//! This module provides efficient memory allocation, registration, and management -//! for RDMA operations with zero-copy semantics and proper cleanup. - -use crate::{RdmaError, RdmaResult}; -use memmap2::MmapMut; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::sync::Arc; -use tracing::{debug, info, warn}; - -/// Memory pool for efficient buffer allocation -pub struct MemoryPool { - /// Pre-allocated memory regions by size - pools: RwLock>>, - /// Total allocated memory in bytes - total_allocated: RwLock, - /// Maximum pool size per buffer size - max_pool_size: usize, - /// Maximum total memory usage - max_total_memory: usize, - /// Statistics - stats: RwLock, -} - -/// Statistics for memory pool -#[derive(Debug, Clone, Default)] -pub struct MemoryPoolStats { - /// Total allocations requested - pub total_allocations: u64, - /// Total deallocations - pub total_deallocations: u64, - /// Cache hits (reused buffers) - pub cache_hits: u64, - /// Cache misses (new allocations) - pub cache_misses: u64, - /// Current active allocations - pub active_allocations: usize, - /// Peak memory usage in bytes - pub peak_memory_usage: usize, -} - -/// A pooled memory buffer -pub struct PooledBuffer { - /// Raw buffer data - data: Vec, - /// Size of the buffer - size: usize, - /// Whether the buffer is currently in use - in_use: bool, - /// Creation timestamp - created_at: std::time::Instant, -} - -impl PooledBuffer { - /// Create new pooled buffer - fn new(size: usize) -> Self { - Self { - data: vec![0u8; size], - size, - in_use: false, - created_at: std::time::Instant::now(), - } - } - - /// Get buffer data as slice - pub fn as_slice(&self) -> &[u8] { - &self.data - } - - /// Get buffer data as mutable slice - pub fn as_mut_slice(&mut self) -> &mut [u8] { - &mut self.data - } - - /// Get buffer size - pub fn size(&self) -> usize { - self.size - } - - /// Get buffer age - pub fn age(&self) -> std::time::Duration { - self.created_at.elapsed() - } - - /// Get raw pointer to buffer data - pub fn as_ptr(&self) -> *const u8 { - self.data.as_ptr() - } - - /// Get mutable raw pointer to buffer data - pub fn as_mut_ptr(&mut self) -> *mut u8 { - self.data.as_mut_ptr() - } -} - -impl MemoryPool { - /// Create new memory pool - pub fn new(max_pool_size: usize, max_total_memory: usize) -> Self { - info!("๐Ÿง  Memory pool initialized: max_pool_size={}, max_total_memory={} bytes", - max_pool_size, max_total_memory); - - Self { - pools: RwLock::new(HashMap::new()), - total_allocated: RwLock::new(0), - max_pool_size, - max_total_memory, - stats: RwLock::new(MemoryPoolStats::default()), - } - } - - /// Allocate buffer from pool - pub fn allocate(&self, size: usize) -> RdmaResult>> { - // Round up to next power of 2 for better pooling - let pool_size = size.next_power_of_two(); - - { - let mut stats = self.stats.write(); - stats.total_allocations += 1; - } - - // Try to get buffer from pool first - { - let mut pools = self.pools.write(); - if let Some(pool) = pools.get_mut(&pool_size) { - // Find available buffer in pool - for buffer in pool.iter_mut() { - if !buffer.in_use { - buffer.in_use = true; - - let mut stats = self.stats.write(); - stats.cache_hits += 1; - stats.active_allocations += 1; - - debug!("๐Ÿ“ฆ Reused buffer from pool: size={}", pool_size); - return Ok(Arc::new(RwLock::new(std::mem::replace( - buffer, - PooledBuffer::new(0) // Placeholder - )))); - } - } - } - } - - // No available buffer in pool, create new one - let total_allocated = *self.total_allocated.read(); - if total_allocated + pool_size > self.max_total_memory { - return Err(RdmaError::ResourceExhausted { - resource: "memory".to_string() - }); - } - - let mut buffer = PooledBuffer::new(pool_size); - buffer.in_use = true; - - // Update allocation tracking - let new_total = { - let mut total = self.total_allocated.write(); - *total += pool_size; - *total - }; - - { - let mut stats = self.stats.write(); - stats.cache_misses += 1; - stats.active_allocations += 1; - if new_total > stats.peak_memory_usage { - stats.peak_memory_usage = new_total; - } - } - - debug!("๐Ÿ†• Allocated new buffer: size={}, total_allocated={}", - pool_size, new_total); - - Ok(Arc::new(RwLock::new(buffer))) - } - - /// Return buffer to pool - pub fn deallocate(&self, buffer: Arc>) -> RdmaResult<()> { - let buffer_size = { - let buf = buffer.read(); - buf.size() - }; - - { - let mut stats = self.stats.write(); - stats.total_deallocations += 1; - stats.active_allocations = stats.active_allocations.saturating_sub(1); - } - - // Try to return buffer to pool - { - let mut pools = self.pools.write(); - let pool = pools.entry(buffer_size).or_insert_with(Vec::new); - - if pool.len() < self.max_pool_size { - // Reset buffer state and return to pool - if let Ok(buf) = Arc::try_unwrap(buffer) { - let mut buf = buf.into_inner(); - buf.in_use = false; - buf.data.fill(0); // Clear data for security - pool.push(buf); - - debug!("โ™ป๏ธ Returned buffer to pool: size={}", buffer_size); - return Ok(()); - } - } - } - - // Pool is full or buffer is still referenced, just track deallocation - { - let mut total = self.total_allocated.write(); - *total = total.saturating_sub(buffer_size); - } - - debug!("๐Ÿ—‘๏ธ Buffer deallocated (not pooled): size={}", buffer_size); - Ok(()) - } - - /// Get memory pool statistics - pub fn stats(&self) -> MemoryPoolStats { - self.stats.read().clone() - } - - /// Get current memory usage - pub fn current_usage(&self) -> usize { - *self.total_allocated.read() - } - - /// Clean up old unused buffers from pools - pub fn cleanup_old_buffers(&self, max_age: std::time::Duration) { - let mut cleaned_count = 0; - let mut cleaned_bytes = 0; - - { - let mut pools = self.pools.write(); - for (size, pool) in pools.iter_mut() { - pool.retain(|buffer| { - if buffer.age() > max_age && !buffer.in_use { - cleaned_count += 1; - cleaned_bytes += size; - false - } else { - true - } - }); - } - } - - if cleaned_count > 0 { - { - let mut total = self.total_allocated.write(); - *total = total.saturating_sub(cleaned_bytes); - } - - info!("๐Ÿงน Cleaned up {} old buffers, freed {} bytes", - cleaned_count, cleaned_bytes); - } - } -} - -/// RDMA-specific memory manager -pub struct RdmaMemoryManager { - /// General purpose memory pool - pool: MemoryPool, - /// Memory-mapped regions for large allocations - mmapped_regions: RwLock>, - /// HugePage allocations (if available) - hugepage_regions: RwLock>, - /// Configuration - config: MemoryConfig, -} - -/// Memory configuration -#[derive(Debug, Clone)] -pub struct MemoryConfig { - /// Use hugepages for large allocations - pub use_hugepages: bool, - /// Hugepage size in bytes - pub hugepage_size: usize, - /// Memory pool settings - pub pool_max_size: usize, - /// Maximum total memory usage - pub max_total_memory: usize, - /// Buffer cleanup interval - pub cleanup_interval_secs: u64, -} - -impl Default for MemoryConfig { - fn default() -> Self { - Self { - use_hugepages: true, - hugepage_size: 2 * 1024 * 1024, // 2MB - pool_max_size: 1000, - max_total_memory: 8 * 1024 * 1024 * 1024, // 8GB - cleanup_interval_secs: 300, // 5 minutes - } - } -} - -/// Memory-mapped region -#[allow(dead_code)] -struct MmapRegion { - mmap: MmapMut, - size: usize, - created_at: std::time::Instant, -} - -/// HugePage memory region -#[allow(dead_code)] -struct HugePageRegion { - addr: *mut u8, - size: usize, - created_at: std::time::Instant, -} - -unsafe impl Send for HugePageRegion {} -unsafe impl Sync for HugePageRegion {} - -impl RdmaMemoryManager { - /// Create new RDMA memory manager - pub fn new(config: MemoryConfig) -> Self { - let pool = MemoryPool::new(config.pool_max_size, config.max_total_memory); - - Self { - pool, - mmapped_regions: RwLock::new(HashMap::new()), - hugepage_regions: RwLock::new(HashMap::new()), - config, - } - } - - /// Allocate memory optimized for RDMA operations - pub fn allocate_rdma_buffer(&self, size: usize) -> RdmaResult { - if size >= self.config.hugepage_size && self.config.use_hugepages { - self.allocate_hugepage_buffer(size) - } else if size >= 64 * 1024 { // Use mmap for large buffers - self.allocate_mmap_buffer(size) - } else { - self.allocate_pool_buffer(size) - } - } - - /// Allocate buffer from memory pool - fn allocate_pool_buffer(&self, size: usize) -> RdmaResult { - let buffer = self.pool.allocate(size)?; - Ok(RdmaBuffer::Pool { buffer, size }) - } - - /// Allocate memory-mapped buffer - fn allocate_mmap_buffer(&self, size: usize) -> RdmaResult { - let mmap = MmapMut::map_anon(size) - .map_err(|e| RdmaError::memory_reg_failed(format!("mmap failed: {}", e)))?; - - let addr = mmap.as_ptr() as u64; - let region = MmapRegion { - mmap, - size, - created_at: std::time::Instant::now(), - }; - - { - let mut regions = self.mmapped_regions.write(); - regions.insert(addr, region); - } - - debug!("๐Ÿ—บ๏ธ Allocated mmap buffer: addr=0x{:x}, size={}", addr, size); - Ok(RdmaBuffer::Mmap { addr, size }) - } - - /// Allocate hugepage buffer (Linux-specific) - fn allocate_hugepage_buffer(&self, size: usize) -> RdmaResult { - #[cfg(target_os = "linux")] - { - use nix::sys::mman::{mmap, MapFlags, ProtFlags}; - - // Round up to hugepage boundary - let aligned_size = (size + self.config.hugepage_size - 1) & !(self.config.hugepage_size - 1); - - let addr = unsafe { - // For anonymous mapping, we can use -1 as the file descriptor - use std::os::fd::BorrowedFd; - let fake_fd = BorrowedFd::borrow_raw(-1); // Anonymous mapping uses -1 - - mmap( - None, // ptr::null_mut() -> None - std::num::NonZero::new(aligned_size).unwrap(), // aligned_size -> NonZero - ProtFlags::PROT_READ | ProtFlags::PROT_WRITE, - MapFlags::MAP_PRIVATE | MapFlags::MAP_ANONYMOUS | MapFlags::MAP_HUGETLB, - Some(&fake_fd), // Use borrowed FD for -1 wrapped in Some - 0, - ) - }; - - match addr { - Ok(addr) => { - let addr_u64 = addr as u64; - let region = HugePageRegion { - addr: addr as *mut u8, - size: aligned_size, - created_at: std::time::Instant::now(), - }; - - { - let mut regions = self.hugepage_regions.write(); - regions.insert(addr_u64, region); - } - - info!("๐Ÿ”ฅ Allocated hugepage buffer: addr=0x{:x}, size={}", addr_u64, aligned_size); - Ok(RdmaBuffer::HugePage { addr: addr_u64, size: aligned_size }) - } - Err(_) => { - warn!("Failed to allocate hugepage buffer, falling back to mmap"); - self.allocate_mmap_buffer(size) - } - } - } - - #[cfg(not(target_os = "linux"))] - { - warn!("HugePages not supported on this platform, using mmap"); - self.allocate_mmap_buffer(size) - } - } - - /// Deallocate RDMA buffer - pub fn deallocate_buffer(&self, buffer: RdmaBuffer) -> RdmaResult<()> { - match buffer { - RdmaBuffer::Pool { buffer, .. } => { - self.pool.deallocate(buffer) - } - RdmaBuffer::Mmap { addr, .. } => { - let mut regions = self.mmapped_regions.write(); - regions.remove(&addr); - debug!("๐Ÿ—‘๏ธ Deallocated mmap buffer: addr=0x{:x}", addr); - Ok(()) - } - RdmaBuffer::HugePage { addr, size } => { - { - let mut regions = self.hugepage_regions.write(); - regions.remove(&addr); - } - - #[cfg(target_os = "linux")] - { - use nix::sys::mman::munmap; - unsafe { - let _ = munmap(addr as *mut std::ffi::c_void, size); - } - } - - debug!("๐Ÿ—‘๏ธ Deallocated hugepage buffer: addr=0x{:x}, size={}", addr, size); - Ok(()) - } - } - } - - /// Get memory manager statistics - pub fn stats(&self) -> MemoryManagerStats { - let pool_stats = self.pool.stats(); - let mmap_count = self.mmapped_regions.read().len(); - let hugepage_count = self.hugepage_regions.read().len(); - - MemoryManagerStats { - pool_stats, - mmap_regions: mmap_count, - hugepage_regions: hugepage_count, - total_memory_usage: self.pool.current_usage(), - } - } - - /// Start background cleanup task - pub async fn start_cleanup_task(&self) -> tokio::task::JoinHandle<()> { - let pool = MemoryPool::new(self.config.pool_max_size, self.config.max_total_memory); - let cleanup_interval = std::time::Duration::from_secs(self.config.cleanup_interval_secs); - - tokio::spawn(async move { - let mut interval = tokio::time::interval( - tokio::time::Duration::from_secs(300) // 5 minutes - ); - - loop { - interval.tick().await; - pool.cleanup_old_buffers(cleanup_interval); - } - }) - } -} - -/// RDMA buffer types -pub enum RdmaBuffer { - /// Buffer from memory pool - Pool { - buffer: Arc>, - size: usize, - }, - /// Memory-mapped buffer - Mmap { - addr: u64, - size: usize, - }, - /// HugePage buffer - HugePage { - addr: u64, - size: usize, - }, -} - -impl RdmaBuffer { - /// Get buffer address - pub fn addr(&self) -> u64 { - match self { - Self::Pool { buffer, .. } => { - buffer.read().as_ptr() as u64 - } - Self::Mmap { addr, .. } => *addr, - Self::HugePage { addr, .. } => *addr, - } - } - - /// Get buffer size - pub fn size(&self) -> usize { - match self { - Self::Pool { size, .. } => *size, - Self::Mmap { size, .. } => *size, - Self::HugePage { size, .. } => *size, - } - } - - /// Get buffer as Vec (copy to avoid lifetime issues) - pub fn to_vec(&self) -> Vec { - match self { - Self::Pool { buffer, .. } => { - buffer.read().as_slice().to_vec() - } - Self::Mmap { addr, size } => { - unsafe { - let slice = std::slice::from_raw_parts(*addr as *const u8, *size); - slice.to_vec() - } - } - Self::HugePage { addr, size } => { - unsafe { - let slice = std::slice::from_raw_parts(*addr as *const u8, *size); - slice.to_vec() - } - } - } - } - - /// Get buffer type name - pub fn buffer_type(&self) -> &'static str { - match self { - Self::Pool { .. } => "pool", - Self::Mmap { .. } => "mmap", - Self::HugePage { .. } => "hugepage", - } - } -} - -/// Memory manager statistics -#[derive(Debug, Clone)] -pub struct MemoryManagerStats { - /// Pool statistics - pub pool_stats: MemoryPoolStats, - /// Number of mmap regions - pub mmap_regions: usize, - /// Number of hugepage regions - pub hugepage_regions: usize, - /// Total memory usage in bytes - pub total_memory_usage: usize, -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_memory_pool_allocation() { - let pool = MemoryPool::new(10, 1024 * 1024); - - let buffer1 = pool.allocate(4096).unwrap(); - let buffer2 = pool.allocate(4096).unwrap(); - - assert_eq!(buffer1.read().size(), 4096); - assert_eq!(buffer2.read().size(), 4096); - - let stats = pool.stats(); - assert_eq!(stats.total_allocations, 2); - assert_eq!(stats.cache_misses, 2); - } - - #[test] - fn test_memory_pool_reuse() { - let pool = MemoryPool::new(10, 1024 * 1024); - - // Allocate and deallocate - let buffer = pool.allocate(4096).unwrap(); - let size = buffer.read().size(); - pool.deallocate(buffer).unwrap(); - - // Allocate again - should reuse - let buffer2 = pool.allocate(4096).unwrap(); - assert_eq!(buffer2.read().size(), size); - - let stats = pool.stats(); - assert_eq!(stats.cache_hits, 1); - } - - #[tokio::test] - async fn test_rdma_memory_manager() { - let config = MemoryConfig::default(); - let manager = RdmaMemoryManager::new(config); - - // Test small buffer (pool) - let small_buffer = manager.allocate_rdma_buffer(1024).unwrap(); - assert_eq!(small_buffer.size(), 1024); - assert_eq!(small_buffer.buffer_type(), "pool"); - - // Test large buffer (mmap) - let large_buffer = manager.allocate_rdma_buffer(128 * 1024).unwrap(); - assert_eq!(large_buffer.size(), 128 * 1024); - assert_eq!(large_buffer.buffer_type(), "mmap"); - - // Clean up - manager.deallocate_buffer(small_buffer).unwrap(); - manager.deallocate_buffer(large_buffer).unwrap(); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/rdma.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/rdma.rs deleted file mode 100644 index 7549a217e..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/rdma.rs +++ /dev/null @@ -1,467 +0,0 @@ -//! RDMA operations and context management -//! -//! This module provides both mock and real RDMA implementations: -//! - Mock implementation for development and testing -//! - Real implementation using libibverbs for production - -use crate::{RdmaResult, RdmaEngineConfig}; -use tracing::{debug, warn, info}; -use parking_lot::RwLock; - -/// RDMA completion status -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum CompletionStatus { - Success, - LocalLengthError, - LocalQpOperationError, - LocalEecOperationError, - LocalProtectionError, - WrFlushError, - MemoryWindowBindError, - BadResponseError, - LocalAccessError, - RemoteInvalidRequestError, - RemoteAccessError, - RemoteOperationError, - TransportRetryCounterExceeded, - RnrRetryCounterExceeded, - LocalRddViolationError, - RemoteInvalidRdRequest, - RemoteAbortedError, - InvalidEecnError, - InvalidEecStateError, - FatalError, - ResponseTimeoutError, - GeneralError, -} - -impl From for CompletionStatus { - fn from(status: u32) -> Self { - match status { - 0 => Self::Success, - 1 => Self::LocalLengthError, - 2 => Self::LocalQpOperationError, - 3 => Self::LocalEecOperationError, - 4 => Self::LocalProtectionError, - 5 => Self::WrFlushError, - 6 => Self::MemoryWindowBindError, - 7 => Self::BadResponseError, - 8 => Self::LocalAccessError, - 9 => Self::RemoteInvalidRequestError, - 10 => Self::RemoteAccessError, - 11 => Self::RemoteOperationError, - 12 => Self::TransportRetryCounterExceeded, - 13 => Self::RnrRetryCounterExceeded, - 14 => Self::LocalRddViolationError, - 15 => Self::RemoteInvalidRdRequest, - 16 => Self::RemoteAbortedError, - 17 => Self::InvalidEecnError, - 18 => Self::InvalidEecStateError, - 19 => Self::FatalError, - 20 => Self::ResponseTimeoutError, - _ => Self::GeneralError, - } - } -} - -/// RDMA operation types -#[derive(Debug, Clone, Copy)] -pub enum RdmaOp { - Read, - Write, - Send, - Receive, - Atomic, -} - -/// RDMA memory region information -#[derive(Debug, Clone)] -pub struct MemoryRegion { - /// Local virtual address - pub addr: u64, - /// Remote key for RDMA operations - pub rkey: u32, - /// Local key for local operations - pub lkey: u32, - /// Size of the memory region - pub size: usize, - /// Whether the region is registered with RDMA hardware - pub registered: bool, -} - -/// RDMA work completion -#[derive(Debug)] -pub struct WorkCompletion { - /// Work request ID - pub wr_id: u64, - /// Completion status - pub status: CompletionStatus, - /// Operation type - pub opcode: RdmaOp, - /// Number of bytes transferred - pub byte_len: u32, - /// Immediate data (if any) - pub imm_data: Option, -} - -/// RDMA context implementation (simplified enum approach) -#[derive(Debug)] -pub enum RdmaContextImpl { - Mock(MockRdmaContext), - // Ucx(UcxRdmaContext), // TODO: Add UCX implementation -} - -/// RDMA device information -#[derive(Debug, Clone)] -pub struct RdmaDeviceInfo { - pub name: String, - pub vendor_id: u32, - pub vendor_part_id: u32, - pub hw_ver: u32, - pub max_mr: u32, - pub max_qp: u32, - pub max_cq: u32, - pub max_mr_size: u64, - pub port_gid: String, - pub port_lid: u16, -} - -/// Main RDMA context -pub struct RdmaContext { - inner: RdmaContextImpl, - #[allow(dead_code)] - config: RdmaEngineConfig, -} - -impl RdmaContext { - /// Create new RDMA context - pub async fn new(config: &RdmaEngineConfig) -> RdmaResult { - let inner = if cfg!(feature = "real-ucx") { - RdmaContextImpl::Mock(MockRdmaContext::new(config).await?) // TODO: Use UCX when ready - } else { - RdmaContextImpl::Mock(MockRdmaContext::new(config).await?) - }; - - Ok(Self { - inner, - config: config.clone(), - }) - } - - /// Register memory for RDMA operations - pub async fn register_memory(&self, addr: u64, size: usize) -> RdmaResult { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.register_memory(addr, size).await, - } - } - - /// Deregister memory region - pub async fn deregister_memory(&self, region: &MemoryRegion) -> RdmaResult<()> { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.deregister_memory(region).await, - } - } - - /// Post RDMA read operation - pub async fn post_read(&self, - local_addr: u64, - remote_addr: u64, - rkey: u32, - size: usize, - wr_id: u64, - ) -> RdmaResult<()> { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.post_read(local_addr, remote_addr, rkey, size, wr_id).await, - } - } - - /// Post RDMA write operation - pub async fn post_write(&self, - local_addr: u64, - remote_addr: u64, - rkey: u32, - size: usize, - wr_id: u64, - ) -> RdmaResult<()> { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.post_write(local_addr, remote_addr, rkey, size, wr_id).await, - } - } - - /// Poll for work completions - pub async fn poll_completion(&self, max_completions: usize) -> RdmaResult> { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.poll_completion(max_completions).await, - } - } - - /// Get device information - pub fn device_info(&self) -> &RdmaDeviceInfo { - match &self.inner { - RdmaContextImpl::Mock(ctx) => ctx.device_info(), - } - } -} - -/// Mock RDMA context for testing and development -#[derive(Debug)] -pub struct MockRdmaContext { - device_info: RdmaDeviceInfo, - registered_regions: RwLock>, - pending_operations: RwLock>, - #[allow(dead_code)] - config: RdmaEngineConfig, -} - -impl MockRdmaContext { - pub async fn new(config: &RdmaEngineConfig) -> RdmaResult { - warn!("๐ŸŸก Using MOCK RDMA implementation - for development only!"); - info!(" Device: {} (mock)", config.device_name); - info!(" Port: {} (mock)", config.port); - - let device_info = RdmaDeviceInfo { - name: config.device_name.clone(), - vendor_id: 0x02c9, // Mellanox mock vendor ID - vendor_part_id: 0x1017, // ConnectX-5 mock part ID - hw_ver: 0, - max_mr: 131072, - max_qp: 262144, - max_cq: 65536, - max_mr_size: 1024 * 1024 * 1024 * 1024, // 1TB mock - port_gid: "fe80:0000:0000:0000:0200:5eff:fe12:3456".to_string(), - port_lid: 1, - }; - - Ok(Self { - device_info, - registered_regions: RwLock::new(Vec::new()), - pending_operations: RwLock::new(Vec::new()), - config: config.clone(), - }) - } -} - -impl MockRdmaContext { - pub async fn register_memory(&self, addr: u64, size: usize) -> RdmaResult { - debug!("๐ŸŸก Mock: Registering memory region addr=0x{:x}, size={}", addr, size); - - // Simulate registration delay - tokio::time::sleep(tokio::time::Duration::from_micros(10)).await; - - let region = MemoryRegion { - addr, - rkey: 0x12345678, // Mock remote key - lkey: 0x87654321, // Mock local key - size, - registered: true, - }; - - self.registered_regions.write().push(region.clone()); - - Ok(region) - } - - pub async fn deregister_memory(&self, region: &MemoryRegion) -> RdmaResult<()> { - debug!("๐ŸŸก Mock: Deregistering memory region rkey=0x{:x}", region.rkey); - - let mut regions = self.registered_regions.write(); - regions.retain(|r| r.rkey != region.rkey); - - Ok(()) - } - - pub async fn post_read(&self, - local_addr: u64, - remote_addr: u64, - rkey: u32, - size: usize, - wr_id: u64, - ) -> RdmaResult<()> { - debug!("๐ŸŸก Mock: RDMA READ local=0x{:x}, remote=0x{:x}, rkey=0x{:x}, size={}", - local_addr, remote_addr, rkey, size); - - // Simulate RDMA read latency (much faster than real network, but realistic for mock) - tokio::time::sleep(tokio::time::Duration::from_nanos(150)).await; - - // Mock data transfer - copy pattern data to local address - let data_ptr = local_addr as *mut u8; - unsafe { - for i in 0..size { - *data_ptr.add(i) = (i % 256) as u8; // Pattern: 0,1,2,...,255,0,1,2... - } - } - - // Create completion - let completion = WorkCompletion { - wr_id, - status: CompletionStatus::Success, - opcode: RdmaOp::Read, - byte_len: size as u32, - imm_data: None, - }; - - self.pending_operations.write().push(completion); - - Ok(()) - } - - pub async fn post_write(&self, - local_addr: u64, - remote_addr: u64, - rkey: u32, - size: usize, - wr_id: u64, - ) -> RdmaResult<()> { - debug!("๐ŸŸก Mock: RDMA WRITE local=0x{:x}, remote=0x{:x}, rkey=0x{:x}, size={}", - local_addr, remote_addr, rkey, size); - - // Simulate RDMA write latency - tokio::time::sleep(tokio::time::Duration::from_nanos(100)).await; - - // Create completion - let completion = WorkCompletion { - wr_id, - status: CompletionStatus::Success, - opcode: RdmaOp::Write, - byte_len: size as u32, - imm_data: None, - }; - - self.pending_operations.write().push(completion); - - Ok(()) - } - - pub async fn poll_completion(&self, max_completions: usize) -> RdmaResult> { - let mut operations = self.pending_operations.write(); - let available = operations.len().min(max_completions); - let completions = operations.drain(..available).collect(); - - Ok(completions) - } - - pub fn device_info(&self) -> &RdmaDeviceInfo { - &self.device_info - } -} - -/// Real RDMA context using libibverbs -#[cfg(feature = "real-ucx")] -pub struct RealRdmaContext { - // Real implementation would contain: - // ibv_context: *mut ibv_context, - // ibv_pd: *mut ibv_pd, - // ibv_cq: *mut ibv_cq, - // ibv_qp: *mut ibv_qp, - device_info: RdmaDeviceInfo, - config: RdmaEngineConfig, -} - -#[cfg(feature = "real-ucx")] -impl RealRdmaContext { - pub async fn new(config: &RdmaEngineConfig) -> RdmaResult { - info!("โœ… Initializing REAL RDMA context for device: {}", config.device_name); - - // Real implementation would: - // 1. Get device list with ibv_get_device_list() - // 2. Find device by name - // 3. Open device with ibv_open_device() - // 4. Create protection domain with ibv_alloc_pd() - // 5. Create completion queue with ibv_create_cq() - // 6. Create queue pair with ibv_create_qp() - // 7. Transition QP to RTS state - - todo!("Real RDMA implementation using libibverbs"); - } -} - -#[cfg(feature = "real-ucx")] -#[async_trait::async_trait] -impl RdmaContextTrait for RealRdmaContext { - async fn register_memory(&self, _addr: u64, _size: usize) -> RdmaResult { - // Real implementation would use ibv_reg_mr() - todo!("Real memory registration") - } - - async fn deregister_memory(&self, _region: &MemoryRegion) -> RdmaResult<()> { - // Real implementation would use ibv_dereg_mr() - todo!("Real memory deregistration") - } - - async fn post_read(&self, - _local_addr: u64, - _remote_addr: u64, - _rkey: u32, - _size: usize, - _wr_id: u64, - ) -> RdmaResult<()> { - // Real implementation would use ibv_post_send() with IBV_WR_RDMA_READ - todo!("Real RDMA read") - } - - async fn post_write(&self, - _local_addr: u64, - _remote_addr: u64, - _rkey: u32, - _size: usize, - _wr_id: u64, - ) -> RdmaResult<()> { - // Real implementation would use ibv_post_send() with IBV_WR_RDMA_WRITE - todo!("Real RDMA write") - } - - async fn poll_completion(&self, _max_completions: usize) -> RdmaResult> { - // Real implementation would use ibv_poll_cq() - todo!("Real completion polling") - } - - fn device_info(&self) -> &RdmaDeviceInfo { - &self.device_info - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_mock_rdma_context() { - let config = RdmaEngineConfig::default(); - let ctx = RdmaContext::new(&config).await.unwrap(); - - // Test device info - let info = ctx.device_info(); - assert_eq!(info.name, "mlx5_0"); - assert!(info.max_mr > 0); - - // Test memory registration - let addr = 0x7f000000u64; - let size = 4096; - let region = ctx.register_memory(addr, size).await.unwrap(); - assert_eq!(region.addr, addr); - assert_eq!(region.size, size); - assert!(region.registered); - - // Test RDMA read - let local_buf = vec![0u8; 1024]; - let local_addr = local_buf.as_ptr() as u64; - let result = ctx.post_read(local_addr, 0x8000000, region.rkey, 1024, 1).await; - assert!(result.is_ok()); - - // Test completion polling - let completions = ctx.poll_completion(10).await.unwrap(); - assert_eq!(completions.len(), 1); - assert_eq!(completions[0].status, CompletionStatus::Success); - - // Test memory deregistration - let result = ctx.deregister_memory(®ion).await; - assert!(result.is_ok()); - } - - #[test] - fn test_completion_status_conversion() { - assert_eq!(CompletionStatus::from(0), CompletionStatus::Success); - assert_eq!(CompletionStatus::from(1), CompletionStatus::LocalLengthError); - assert_eq!(CompletionStatus::from(999), CompletionStatus::GeneralError); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/session.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/session.rs deleted file mode 100644 index fa089c72a..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/session.rs +++ /dev/null @@ -1,587 +0,0 @@ -//! Session management for RDMA operations -//! -//! This module manages the lifecycle of RDMA sessions, including creation, -//! storage, expiration, and cleanup of resources. - -use crate::{RdmaError, RdmaResult, rdma::MemoryRegion}; -use parking_lot::RwLock; -use std::collections::HashMap; -use std::sync::Arc; -use tokio::time::{Duration, Instant}; -use tracing::{debug, info}; -// use uuid::Uuid; // Unused for now - -/// RDMA session state -#[derive(Debug, Clone)] -pub struct RdmaSession { - /// Unique session identifier - pub id: String, - /// SeaweedFS volume ID - pub volume_id: u32, - /// SeaweedFS needle ID - pub needle_id: u64, - /// Remote memory address - pub remote_addr: u64, - /// Remote key for RDMA access - pub remote_key: u32, - /// Transfer size in bytes - pub transfer_size: u64, - /// Local data buffer - pub buffer: Vec, - /// RDMA memory region - pub memory_region: MemoryRegion, - /// Session creation time - pub created_at: Instant, - /// Session expiration time - pub expires_at: Instant, - /// Current session state - pub state: SessionState, - /// Operation statistics - pub stats: SessionStats, -} - -/// Session state enum -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SessionState { - /// Session created but not yet active - Created, - /// RDMA operation in progress - Active, - /// Operation completed successfully - Completed, - /// Operation failed - Failed, - /// Session expired - Expired, - /// Session being cleaned up - CleaningUp, -} - -/// Session operation statistics -#[derive(Debug, Clone, Default)] -pub struct SessionStats { - /// Number of RDMA operations performed - pub operations_count: u64, - /// Total bytes transferred - pub bytes_transferred: u64, - /// Time spent in RDMA operations (nanoseconds) - pub rdma_time_ns: u64, - /// Number of completion polling attempts - pub poll_attempts: u64, - /// Time of last operation - pub last_operation_at: Option, -} - -impl RdmaSession { - /// Create a new RDMA session - pub fn new( - id: String, - volume_id: u32, - needle_id: u64, - remote_addr: u64, - remote_key: u32, - transfer_size: u64, - buffer: Vec, - memory_region: MemoryRegion, - timeout: Duration, - ) -> Self { - let now = Instant::now(); - - Self { - id, - volume_id, - needle_id, - remote_addr, - remote_key, - transfer_size, - buffer, - memory_region, - created_at: now, - expires_at: now + timeout, - state: SessionState::Created, - stats: SessionStats::default(), - } - } - - /// Check if session has expired - pub fn is_expired(&self) -> bool { - Instant::now() > self.expires_at - } - - /// Get session age in seconds - pub fn age_secs(&self) -> f64 { - self.created_at.elapsed().as_secs_f64() - } - - /// Get time until expiration in seconds - pub fn time_to_expiration_secs(&self) -> f64 { - if self.is_expired() { - 0.0 - } else { - (self.expires_at - Instant::now()).as_secs_f64() - } - } - - /// Update session state - pub fn set_state(&mut self, state: SessionState) { - debug!("Session {} state: {:?} -> {:?}", self.id, self.state, state); - self.state = state; - } - - /// Record RDMA operation statistics - pub fn record_operation(&mut self, bytes_transferred: u64, duration_ns: u64) { - self.stats.operations_count += 1; - self.stats.bytes_transferred += bytes_transferred; - self.stats.rdma_time_ns += duration_ns; - self.stats.last_operation_at = Some(Instant::now()); - } - - /// Get average operation latency in nanoseconds - pub fn avg_operation_latency_ns(&self) -> u64 { - if self.stats.operations_count > 0 { - self.stats.rdma_time_ns / self.stats.operations_count - } else { - 0 - } - } - - /// Get throughput in bytes per second - pub fn throughput_bps(&self) -> f64 { - let age_secs = self.age_secs(); - if age_secs > 0.0 { - self.stats.bytes_transferred as f64 / age_secs - } else { - 0.0 - } - } -} - -/// Session manager for handling multiple concurrent RDMA sessions -pub struct SessionManager { - /// Active sessions - sessions: Arc>>>>, - /// Maximum number of concurrent sessions - max_sessions: usize, - /// Default session timeout - #[allow(dead_code)] - default_timeout: Duration, - /// Cleanup task handle - cleanup_task: RwLock>>, - /// Shutdown flag - shutdown_flag: Arc>, - /// Statistics - stats: Arc>, -} - -/// Session manager statistics -#[derive(Debug, Clone, Default)] -pub struct SessionManagerStats { - /// Total sessions created - pub total_sessions_created: u64, - /// Total sessions completed - pub total_sessions_completed: u64, - /// Total sessions failed - pub total_sessions_failed: u64, - /// Total sessions expired - pub total_sessions_expired: u64, - /// Total bytes transferred across all sessions - pub total_bytes_transferred: u64, - /// Manager start time - pub started_at: Option, -} - -impl SessionManager { - /// Create new session manager - pub fn new(max_sessions: usize, default_timeout: Duration) -> Self { - info!("๐ŸŽฏ Session manager initialized: max_sessions={}, timeout={:?}", - max_sessions, default_timeout); - - let mut stats = SessionManagerStats::default(); - stats.started_at = Some(Instant::now()); - - Self { - sessions: Arc::new(RwLock::new(HashMap::new())), - max_sessions, - default_timeout, - cleanup_task: RwLock::new(None), - shutdown_flag: Arc::new(RwLock::new(false)), - stats: Arc::new(RwLock::new(stats)), - } - } - - /// Create a new RDMA session - pub async fn create_session( - &self, - session_id: String, - volume_id: u32, - needle_id: u64, - remote_addr: u64, - remote_key: u32, - transfer_size: u64, - buffer: Vec, - memory_region: MemoryRegion, - timeout: chrono::Duration, - ) -> RdmaResult>> { - // Check session limit - { - let sessions = self.sessions.read(); - if sessions.len() >= self.max_sessions { - return Err(RdmaError::TooManySessions { - max_sessions: self.max_sessions - }); - } - - // Check if session already exists - if sessions.contains_key(&session_id) { - return Err(RdmaError::invalid_request( - format!("Session {} already exists", session_id) - )); - } - } - - let timeout_duration = Duration::from_millis(timeout.num_milliseconds().max(1) as u64); - - let session = Arc::new(RwLock::new(RdmaSession::new( - session_id.clone(), - volume_id, - needle_id, - remote_addr, - remote_key, - transfer_size, - buffer, - memory_region, - timeout_duration, - ))); - - // Store session - { - let mut sessions = self.sessions.write(); - sessions.insert(session_id.clone(), session.clone()); - } - - // Update stats - { - let mut stats = self.stats.write(); - stats.total_sessions_created += 1; - } - - info!("๐Ÿ“ฆ Created session {}: volume={}, needle={}, size={}", - session_id, volume_id, needle_id, transfer_size); - - Ok(session) - } - - /// Get session by ID - pub async fn get_session(&self, session_id: &str) -> RdmaResult>> { - let sessions = self.sessions.read(); - match sessions.get(session_id) { - Some(session) => { - if session.read().is_expired() { - Err(RdmaError::SessionExpired { - session_id: session_id.to_string() - }) - } else { - Ok(session.clone()) - } - } - None => Err(RdmaError::SessionNotFound { - session_id: session_id.to_string() - }), - } - } - - /// Remove and cleanup session - pub async fn remove_session(&self, session_id: &str) -> RdmaResult<()> { - let session = { - let mut sessions = self.sessions.write(); - sessions.remove(session_id) - }; - - if let Some(session) = session { - let session_data = session.read(); - info!("๐Ÿ—‘๏ธ Removed session {}: stats={:?}", session_id, session_data.stats); - - // Update manager stats - { - let mut stats = self.stats.write(); - match session_data.state { - SessionState::Completed => stats.total_sessions_completed += 1, - SessionState::Failed => stats.total_sessions_failed += 1, - SessionState::Expired => stats.total_sessions_expired += 1, - _ => {} - } - stats.total_bytes_transferred += session_data.stats.bytes_transferred; - } - - Ok(()) - } else { - Err(RdmaError::SessionNotFound { - session_id: session_id.to_string() - }) - } - } - - /// Get active session count - pub async fn active_session_count(&self) -> usize { - self.sessions.read().len() - } - - /// Get maximum sessions allowed - pub fn max_sessions(&self) -> usize { - self.max_sessions - } - - /// List active sessions - pub async fn list_sessions(&self) -> Vec { - self.sessions.read().keys().cloned().collect() - } - - /// Get session statistics - pub async fn get_session_stats(&self, session_id: &str) -> RdmaResult { - let session = self.get_session(session_id).await?; - let stats = { - let session_data = session.read(); - session_data.stats.clone() - }; - Ok(stats) - } - - /// Get manager statistics - pub fn get_manager_stats(&self) -> SessionManagerStats { - self.stats.read().clone() - } - - /// Start background cleanup task - pub async fn start_cleanup_task(&self) { - info!("๐Ÿ“‹ Session cleanup task initialized"); - - let sessions = Arc::clone(&self.sessions); - let shutdown_flag = Arc::clone(&self.shutdown_flag); - let stats = Arc::clone(&self.stats); - - let task = tokio::spawn(async move { - let mut interval = tokio::time::interval(Duration::from_secs(30)); // Check every 30 seconds - - loop { - interval.tick().await; - - // Check shutdown flag - if *shutdown_flag.read() { - debug!("๐Ÿ›‘ Session cleanup task shutting down"); - break; - } - - let now = Instant::now(); - let mut expired_sessions = Vec::new(); - - // Find expired sessions - { - let sessions_guard = sessions.read(); - for (session_id, session) in sessions_guard.iter() { - if now > session.read().expires_at { - expired_sessions.push(session_id.clone()); - } - } - } - - // Remove expired sessions - if !expired_sessions.is_empty() { - let mut sessions_guard = sessions.write(); - let mut stats_guard = stats.write(); - - for session_id in expired_sessions { - if let Some(session) = sessions_guard.remove(&session_id) { - let session_data = session.read(); - info!("๐Ÿ—‘๏ธ Cleaned up expired session: {} (volume={}, needle={})", - session_id, session_data.volume_id, session_data.needle_id); - stats_guard.total_sessions_expired += 1; - } - } - - debug!("๐Ÿ“Š Active sessions: {}", sessions_guard.len()); - } - } - }); - - *self.cleanup_task.write() = Some(task); - } - - /// Shutdown session manager - pub async fn shutdown(&self) { - info!("๐Ÿ›‘ Shutting down session manager"); - *self.shutdown_flag.write() = true; - - // Wait for cleanup task to finish - if let Some(task) = self.cleanup_task.write().take() { - let _ = task.await; - } - - // Clean up all remaining sessions - let session_ids: Vec = { - self.sessions.read().keys().cloned().collect() - }; - - for session_id in session_ids { - let _ = self.remove_session(&session_id).await; - } - - let final_stats = self.get_manager_stats(); - info!("๐Ÿ“ˆ Final session manager stats: {:?}", final_stats); - } - - /// Force cleanup of all sessions (for testing) - #[cfg(test)] - pub async fn cleanup_all_sessions(&self) { - let session_ids: Vec = { - self.sessions.read().keys().cloned().collect() - }; - - for session_id in session_ids { - let _ = self.remove_session(&session_id).await; - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::rdma::MemoryRegion; - - #[tokio::test] - async fn test_session_creation() { - let manager = SessionManager::new(10, Duration::from_secs(60)); - - let memory_region = MemoryRegion { - addr: 0x1000, - rkey: 0x12345678, - lkey: 0x87654321, - size: 4096, - registered: true, - }; - - let session = manager.create_session( - "test-session".to_string(), - 1, - 100, - 0x2000, - 0xabcd, - 4096, - vec![0; 4096], - memory_region, - chrono::Duration::seconds(60), - ).await.unwrap(); - - let session_data = session.read(); - assert_eq!(session_data.id, "test-session"); - assert_eq!(session_data.volume_id, 1); - assert_eq!(session_data.needle_id, 100); - assert_eq!(session_data.state, SessionState::Created); - assert!(!session_data.is_expired()); - } - - #[tokio::test] - async fn test_session_expiration() { - let manager = SessionManager::new(10, Duration::from_millis(10)); - - let memory_region = MemoryRegion { - addr: 0x1000, - rkey: 0x12345678, - lkey: 0x87654321, - size: 4096, - registered: true, - }; - - let _session = manager.create_session( - "expire-test".to_string(), - 1, - 100, - 0x2000, - 0xabcd, - 4096, - vec![0; 4096], - memory_region, - chrono::Duration::milliseconds(10), - ).await.unwrap(); - - // Wait for expiration - tokio::time::sleep(Duration::from_millis(20)).await; - - let result = manager.get_session("expire-test").await; - assert!(matches!(result, Err(RdmaError::SessionExpired { .. }))); - } - - #[tokio::test] - async fn test_session_limit() { - let manager = SessionManager::new(2, Duration::from_secs(60)); - - let memory_region = MemoryRegion { - addr: 0x1000, - rkey: 0x12345678, - lkey: 0x87654321, - size: 4096, - registered: true, - }; - - // Create first session - let _session1 = manager.create_session( - "session1".to_string(), - 1, 100, 0x2000, 0xabcd, 4096, - vec![0; 4096], - memory_region.clone(), - chrono::Duration::seconds(60), - ).await.unwrap(); - - // Create second session - let _session2 = manager.create_session( - "session2".to_string(), - 1, 101, 0x3000, 0xabcd, 4096, - vec![0; 4096], - memory_region.clone(), - chrono::Duration::seconds(60), - ).await.unwrap(); - - // Third session should fail - let result = manager.create_session( - "session3".to_string(), - 1, 102, 0x4000, 0xabcd, 4096, - vec![0; 4096], - memory_region, - chrono::Duration::seconds(60), - ).await; - - assert!(matches!(result, Err(RdmaError::TooManySessions { .. }))); - } - - #[tokio::test] - async fn test_session_stats() { - let manager = SessionManager::new(10, Duration::from_secs(60)); - - let memory_region = MemoryRegion { - addr: 0x1000, - rkey: 0x12345678, - lkey: 0x87654321, - size: 4096, - registered: true, - }; - - let session = manager.create_session( - "stats-test".to_string(), - 1, 100, 0x2000, 0xabcd, 4096, - vec![0; 4096], - memory_region, - chrono::Duration::seconds(60), - ).await.unwrap(); - - // Simulate some operations - now using proper interior mutability - { - let mut session_data = session.write(); - session_data.record_operation(1024, 1000000); // 1KB in 1ms - session_data.record_operation(2048, 2000000); // 2KB in 2ms - } - - let stats = manager.get_session_stats("stats-test").await.unwrap(); - assert_eq!(stats.operations_count, 2); - assert_eq!(stats.bytes_transferred, 3072); - assert_eq!(stats.rdma_time_ns, 3000000); - } -} diff --git a/seaweedfs-rdma-sidecar/rdma-engine/src/ucx.rs b/seaweedfs-rdma-sidecar/rdma-engine/src/ucx.rs deleted file mode 100644 index 901149858..000000000 --- a/seaweedfs-rdma-sidecar/rdma-engine/src/ucx.rs +++ /dev/null @@ -1,606 +0,0 @@ -//! UCX (Unified Communication X) FFI bindings and high-level wrapper -//! -//! UCX is a superior alternative to direct libibverbs for RDMA programming. -//! It provides production-proven abstractions and automatic transport selection. -//! -//! References: -//! - UCX Documentation: https://openucx.readthedocs.io/ -//! - UCX GitHub: https://github.com/openucx/ucx -//! - UCX Paper: "UCX: an open source framework for HPC network APIs and beyond" - -use crate::{RdmaError, RdmaResult}; -use libc::{c_char, c_int, c_void, size_t}; -use libloading::{Library, Symbol}; -use parking_lot::Mutex; -use std::collections::HashMap; -use std::ffi::CStr; -use std::ptr; -use std::sync::Arc; -use tracing::{debug, info, warn, error}; - -/// UCX context handle -pub type UcpContext = *mut c_void; -/// UCX worker handle -pub type UcpWorker = *mut c_void; -/// UCX endpoint handle -pub type UcpEp = *mut c_void; -/// UCX memory handle -pub type UcpMem = *mut c_void; -/// UCX request handle -pub type UcpRequest = *mut c_void; - -/// UCX configuration parameters -#[repr(C)] -pub struct UcpParams { - pub field_mask: u64, - pub features: u64, - pub request_size: size_t, - pub request_init: extern "C" fn(*mut c_void), - pub request_cleanup: extern "C" fn(*mut c_void), - pub tag_sender_mask: u64, -} - -/// UCX worker parameters -#[repr(C)] -pub struct UcpWorkerParams { - pub field_mask: u64, - pub thread_mode: c_int, - pub cpu_mask: u64, - pub events: c_int, - pub user_data: *mut c_void, -} - -/// UCX endpoint parameters -#[repr(C)] -pub struct UcpEpParams { - pub field_mask: u64, - pub address: *const c_void, - pub flags: u64, - pub sock_addr: *const c_void, - pub err_handler: UcpErrHandler, - pub user_data: *mut c_void, -} - -/// UCX memory mapping parameters -#[repr(C)] -pub struct UcpMemMapParams { - pub field_mask: u64, - pub address: *mut c_void, - pub length: size_t, - pub flags: u64, - pub prot: c_int, -} - -/// UCX error handler callback -pub type UcpErrHandler = extern "C" fn( - arg: *mut c_void, - ep: UcpEp, - status: c_int, -); - -/// UCX request callback -pub type UcpSendCallback = extern "C" fn( - request: *mut c_void, - status: c_int, - user_data: *mut c_void, -); - -/// UCX feature flags -pub const UCP_FEATURE_TAG: u64 = 1 << 0; -pub const UCP_FEATURE_RMA: u64 = 1 << 1; -pub const UCP_FEATURE_ATOMIC32: u64 = 1 << 2; -pub const UCP_FEATURE_ATOMIC64: u64 = 1 << 3; -pub const UCP_FEATURE_WAKEUP: u64 = 1 << 4; -pub const UCP_FEATURE_STREAM: u64 = 1 << 5; - -/// UCX parameter field masks -pub const UCP_PARAM_FIELD_FEATURES: u64 = 1 << 0; -pub const UCP_PARAM_FIELD_REQUEST_SIZE: u64 = 1 << 1; -pub const UCP_PARAM_FIELD_REQUEST_INIT: u64 = 1 << 2; -pub const UCP_PARAM_FIELD_REQUEST_CLEANUP: u64 = 1 << 3; -pub const UCP_PARAM_FIELD_TAG_SENDER_MASK: u64 = 1 << 4; - -pub const UCP_WORKER_PARAM_FIELD_THREAD_MODE: u64 = 1 << 0; -pub const UCP_WORKER_PARAM_FIELD_CPU_MASK: u64 = 1 << 1; -pub const UCP_WORKER_PARAM_FIELD_EVENTS: u64 = 1 << 2; -pub const UCP_WORKER_PARAM_FIELD_USER_DATA: u64 = 1 << 3; - -pub const UCP_EP_PARAM_FIELD_REMOTE_ADDRESS: u64 = 1 << 0; -pub const UCP_EP_PARAM_FIELD_FLAGS: u64 = 1 << 1; -pub const UCP_EP_PARAM_FIELD_SOCK_ADDR: u64 = 1 << 2; -pub const UCP_EP_PARAM_FIELD_ERR_HANDLER: u64 = 1 << 3; -pub const UCP_EP_PARAM_FIELD_USER_DATA: u64 = 1 << 4; - -pub const UCP_MEM_MAP_PARAM_FIELD_ADDRESS: u64 = 1 << 0; -pub const UCP_MEM_MAP_PARAM_FIELD_LENGTH: u64 = 1 << 1; -pub const UCP_MEM_MAP_PARAM_FIELD_FLAGS: u64 = 1 << 2; -pub const UCP_MEM_MAP_PARAM_FIELD_PROT: u64 = 1 << 3; - -/// UCX status codes -pub const UCS_OK: c_int = 0; -pub const UCS_INPROGRESS: c_int = 1; -pub const UCS_ERR_NO_MESSAGE: c_int = -1; -pub const UCS_ERR_NO_RESOURCE: c_int = -2; -pub const UCS_ERR_IO_ERROR: c_int = -3; -pub const UCS_ERR_NO_MEMORY: c_int = -4; -pub const UCS_ERR_INVALID_PARAM: c_int = -5; -pub const UCS_ERR_UNREACHABLE: c_int = -6; -pub const UCS_ERR_INVALID_ADDR: c_int = -7; -pub const UCS_ERR_NOT_IMPLEMENTED: c_int = -8; -pub const UCS_ERR_MESSAGE_TRUNCATED: c_int = -9; -pub const UCS_ERR_NO_PROGRESS: c_int = -10; -pub const UCS_ERR_BUFFER_TOO_SMALL: c_int = -11; -pub const UCS_ERR_NO_ELEM: c_int = -12; -pub const UCS_ERR_SOME_CONNECTS_FAILED: c_int = -13; -pub const UCS_ERR_NO_DEVICE: c_int = -14; -pub const UCS_ERR_BUSY: c_int = -15; -pub const UCS_ERR_CANCELED: c_int = -16; -pub const UCS_ERR_SHMEM_SEGMENT: c_int = -17; -pub const UCS_ERR_ALREADY_EXISTS: c_int = -18; -pub const UCS_ERR_OUT_OF_RANGE: c_int = -19; -pub const UCS_ERR_TIMED_OUT: c_int = -20; - -/// UCX memory protection flags -pub const UCP_MEM_MAP_NONBLOCK: u64 = 1 << 0; -pub const UCP_MEM_MAP_ALLOCATE: u64 = 1 << 1; -pub const UCP_MEM_MAP_FIXED: u64 = 1 << 2; - -/// UCX FFI function signatures -pub struct UcxApi { - pub ucp_init: Symbol<'static, unsafe extern "C" fn(*const UcpParams, *const c_void, *mut UcpContext) -> c_int>, - pub ucp_cleanup: Symbol<'static, unsafe extern "C" fn(UcpContext)>, - pub ucp_worker_create: Symbol<'static, unsafe extern "C" fn(UcpContext, *const UcpWorkerParams, *mut UcpWorker) -> c_int>, - pub ucp_worker_destroy: Symbol<'static, unsafe extern "C" fn(UcpWorker)>, - pub ucp_ep_create: Symbol<'static, unsafe extern "C" fn(UcpWorker, *const UcpEpParams, *mut UcpEp) -> c_int>, - pub ucp_ep_destroy: Symbol<'static, unsafe extern "C" fn(UcpEp)>, - pub ucp_mem_map: Symbol<'static, unsafe extern "C" fn(UcpContext, *const UcpMemMapParams, *mut UcpMem) -> c_int>, - pub ucp_mem_unmap: Symbol<'static, unsafe extern "C" fn(UcpContext, UcpMem) -> c_int>, - pub ucp_put_nb: Symbol<'static, unsafe extern "C" fn(UcpEp, *const c_void, size_t, u64, u64, UcpSendCallback) -> UcpRequest>, - pub ucp_get_nb: Symbol<'static, unsafe extern "C" fn(UcpEp, *mut c_void, size_t, u64, u64, UcpSendCallback) -> UcpRequest>, - pub ucp_worker_progress: Symbol<'static, unsafe extern "C" fn(UcpWorker) -> c_int>, - pub ucp_request_check_status: Symbol<'static, unsafe extern "C" fn(UcpRequest) -> c_int>, - pub ucp_request_free: Symbol<'static, unsafe extern "C" fn(UcpRequest)>, - pub ucp_worker_get_address: Symbol<'static, unsafe extern "C" fn(UcpWorker, *mut *mut c_void, *mut size_t) -> c_int>, - pub ucp_worker_release_address: Symbol<'static, unsafe extern "C" fn(UcpWorker, *mut c_void)>, - pub ucs_status_string: Symbol<'static, unsafe extern "C" fn(c_int) -> *const c_char>, -} - -impl UcxApi { - /// Load UCX library and resolve symbols - pub fn load() -> RdmaResult { - info!("๐Ÿ”— Loading UCX library"); - - // Try to load UCX library - let lib_names = [ - "libucp.so.0", // Most common - "libucp.so", // Generic - "libucp.dylib", // macOS - "/usr/lib/x86_64-linux-gnu/libucp.so.0", // Ubuntu/Debian - "/usr/lib64/libucp.so.0", // RHEL/CentOS - ]; - - let library = lib_names.iter() - .find_map(|name| { - debug!("Trying to load UCX library: {}", name); - match unsafe { Library::new(name) } { - Ok(lib) => { - info!("โœ… Successfully loaded UCX library: {}", name); - Some(lib) - } - Err(e) => { - debug!("Failed to load {}: {}", name, e); - None - } - } - }) - .ok_or_else(|| RdmaError::context_init_failed("UCX library not found"))?; - - // Leak the library to get 'static lifetime for symbols - let library: &'static Library = Box::leak(Box::new(library)); - - unsafe { - Ok(UcxApi { - ucp_init: library.get(b"ucp_init") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_init symbol: {}", e)))?, - ucp_cleanup: library.get(b"ucp_cleanup") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_cleanup symbol: {}", e)))?, - ucp_worker_create: library.get(b"ucp_worker_create") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_worker_create symbol: {}", e)))?, - ucp_worker_destroy: library.get(b"ucp_worker_destroy") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_worker_destroy symbol: {}", e)))?, - ucp_ep_create: library.get(b"ucp_ep_create") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_ep_create symbol: {}", e)))?, - ucp_ep_destroy: library.get(b"ucp_ep_destroy") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_ep_destroy symbol: {}", e)))?, - ucp_mem_map: library.get(b"ucp_mem_map") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_mem_map symbol: {}", e)))?, - ucp_mem_unmap: library.get(b"ucp_mem_unmap") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_mem_unmap symbol: {}", e)))?, - ucp_put_nb: library.get(b"ucp_put_nb") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_put_nb symbol: {}", e)))?, - ucp_get_nb: library.get(b"ucp_get_nb") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_get_nb symbol: {}", e)))?, - ucp_worker_progress: library.get(b"ucp_worker_progress") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_worker_progress symbol: {}", e)))?, - ucp_request_check_status: library.get(b"ucp_request_check_status") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_request_check_status symbol: {}", e)))?, - ucp_request_free: library.get(b"ucp_request_free") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_request_free symbol: {}", e)))?, - ucp_worker_get_address: library.get(b"ucp_worker_get_address") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_worker_get_address symbol: {}", e)))?, - ucp_worker_release_address: library.get(b"ucp_worker_release_address") - .map_err(|e| RdmaError::context_init_failed(format!("ucp_worker_release_address symbol: {}", e)))?, - ucs_status_string: library.get(b"ucs_status_string") - .map_err(|e| RdmaError::context_init_failed(format!("ucs_status_string symbol: {}", e)))?, - }) - } - } - - /// Convert UCX status code to human-readable string - pub fn status_string(&self, status: c_int) -> String { - unsafe { - let c_str = (self.ucs_status_string)(status); - if c_str.is_null() { - format!("Unknown status: {}", status) - } else { - CStr::from_ptr(c_str).to_string_lossy().to_string() - } - } - } -} - -/// High-level UCX context wrapper -pub struct UcxContext { - api: Arc, - context: UcpContext, - worker: UcpWorker, - worker_address: Vec, - endpoints: Mutex>, - memory_regions: Mutex>, -} - -impl UcxContext { - /// Initialize UCX context with RMA support - pub async fn new() -> RdmaResult { - info!("๐Ÿš€ Initializing UCX context for RDMA operations"); - - let api = Arc::new(UcxApi::load()?); - - // Initialize UCP context - let params = UcpParams { - field_mask: UCP_PARAM_FIELD_FEATURES, - features: UCP_FEATURE_RMA | UCP_FEATURE_WAKEUP, - request_size: 0, - request_init: request_init_cb, - request_cleanup: request_cleanup_cb, - tag_sender_mask: 0, - }; - - let mut context = ptr::null_mut(); - let status = unsafe { (api.ucp_init)(¶ms, ptr::null(), &mut context) }; - if status != UCS_OK { - return Err(RdmaError::context_init_failed(format!( - "ucp_init failed: {} ({})", - api.status_string(status), status - ))); - } - - info!("โœ… UCX context initialized successfully"); - - // Create worker - let worker_params = UcpWorkerParams { - field_mask: UCP_WORKER_PARAM_FIELD_THREAD_MODE, - thread_mode: 0, // Single-threaded - cpu_mask: 0, - events: 0, - user_data: ptr::null_mut(), - }; - - let mut worker = ptr::null_mut(); - let status = unsafe { (api.ucp_worker_create)(context, &worker_params, &mut worker) }; - if status != UCS_OK { - unsafe { (api.ucp_cleanup)(context) }; - return Err(RdmaError::context_init_failed(format!( - "ucp_worker_create failed: {} ({})", - api.status_string(status), status - ))); - } - - info!("โœ… UCX worker created successfully"); - - // Get worker address for connection establishment - let mut address_ptr = ptr::null_mut(); - let mut address_len = 0; - let status = unsafe { (api.ucp_worker_get_address)(worker, &mut address_ptr, &mut address_len) }; - if status != UCS_OK { - unsafe { - (api.ucp_worker_destroy)(worker); - (api.ucp_cleanup)(context); - } - return Err(RdmaError::context_init_failed(format!( - "ucp_worker_get_address failed: {} ({})", - api.status_string(status), status - ))); - } - - let worker_address = unsafe { - std::slice::from_raw_parts(address_ptr as *const u8, address_len).to_vec() - }; - - unsafe { (api.ucp_worker_release_address)(worker, address_ptr) }; - - info!("โœ… UCX worker address obtained ({} bytes)", worker_address.len()); - - Ok(UcxContext { - api, - context, - worker, - worker_address, - endpoints: Mutex::new(HashMap::new()), - memory_regions: Mutex::new(HashMap::new()), - }) - } - - /// Map memory for RDMA operations - pub async fn map_memory(&self, addr: u64, size: usize) -> RdmaResult { - debug!("๐Ÿ“ Mapping memory for RDMA: addr=0x{:x}, size={}", addr, size); - - let params = UcpMemMapParams { - field_mask: UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH, - address: addr as *mut c_void, - length: size, - flags: 0, - prot: libc::PROT_READ | libc::PROT_WRITE, - }; - - let mut mem_handle = ptr::null_mut(); - let status = unsafe { (self.api.ucp_mem_map)(self.context, ¶ms, &mut mem_handle) }; - - if status != UCS_OK { - return Err(RdmaError::memory_reg_failed(format!( - "ucp_mem_map failed: {} ({})", - self.api.status_string(status), status - ))); - } - - // Store memory handle for cleanup - { - let mut regions = self.memory_regions.lock(); - regions.insert(addr, mem_handle); - } - - info!("โœ… Memory mapped successfully: addr=0x{:x}, size={}", addr, size); - Ok(addr) // Return the same address as remote key equivalent - } - - /// Unmap memory - pub async fn unmap_memory(&self, addr: u64) -> RdmaResult<()> { - debug!("๐Ÿ—‘๏ธ Unmapping memory: addr=0x{:x}", addr); - - let mem_handle = { - let mut regions = self.memory_regions.lock(); - regions.remove(&addr) - }; - - if let Some(handle) = mem_handle { - let status = unsafe { (self.api.ucp_mem_unmap)(self.context, handle) }; - if status != UCS_OK { - warn!("ucp_mem_unmap failed: {} ({})", - self.api.status_string(status), status); - } - } - - Ok(()) - } - - /// Perform RDMA GET (read from remote memory) - pub async fn get(&self, local_addr: u64, remote_addr: u64, size: usize) -> RdmaResult<()> { - debug!("๐Ÿ“ฅ RDMA GET: local=0x{:x}, remote=0x{:x}, size={}", - local_addr, remote_addr, size); - - // For now, use a simple synchronous approach - // In production, this would be properly async with completion callbacks - - // Find or create endpoint (simplified - would need proper address resolution) - let ep = self.get_or_create_endpoint("default").await?; - - let request = unsafe { - (self.api.ucp_get_nb)( - ep, - local_addr as *mut c_void, - size, - remote_addr, - 0, // No remote key needed with UCX - get_completion_cb, - ) - }; - - // Wait for completion - if !request.is_null() { - loop { - let status = unsafe { (self.api.ucp_request_check_status)(request) }; - if status != UCS_INPROGRESS { - unsafe { (self.api.ucp_request_free)(request) }; - if status == UCS_OK { - break; - } else { - return Err(RdmaError::operation_failed( - "RDMA GET", status - )); - } - } - - // Progress the worker - unsafe { (self.api.ucp_worker_progress)(self.worker) }; - tokio::task::yield_now().await; - } - } - - info!("โœ… RDMA GET completed successfully"); - Ok(()) - } - - /// Perform RDMA PUT (write to remote memory) - pub async fn put(&self, local_addr: u64, remote_addr: u64, size: usize) -> RdmaResult<()> { - debug!("๐Ÿ“ค RDMA PUT: local=0x{:x}, remote=0x{:x}, size={}", - local_addr, remote_addr, size); - - let ep = self.get_or_create_endpoint("default").await?; - - let request = unsafe { - (self.api.ucp_put_nb)( - ep, - local_addr as *const c_void, - size, - remote_addr, - 0, // No remote key needed with UCX - put_completion_cb, - ) - }; - - // Wait for completion (same pattern as GET) - if !request.is_null() { - loop { - let status = unsafe { (self.api.ucp_request_check_status)(request) }; - if status != UCS_INPROGRESS { - unsafe { (self.api.ucp_request_free)(request) }; - if status == UCS_OK { - break; - } else { - return Err(RdmaError::operation_failed( - "RDMA PUT", status - )); - } - } - - unsafe { (self.api.ucp_worker_progress)(self.worker) }; - tokio::task::yield_now().await; - } - } - - info!("โœ… RDMA PUT completed successfully"); - Ok(()) - } - - /// Get worker address for connection establishment - pub fn worker_address(&self) -> &[u8] { - &self.worker_address - } - - /// Create endpoint for communication (simplified version) - async fn get_or_create_endpoint(&self, key: &str) -> RdmaResult { - let mut endpoints = self.endpoints.lock(); - - if let Some(&ep) = endpoints.get(key) { - return Ok(ep); - } - - // For simplicity, create a dummy endpoint - // In production, this would use actual peer address - let ep_params = UcpEpParams { - field_mask: 0, // Simplified for mock - address: ptr::null(), - flags: 0, - sock_addr: ptr::null(), - err_handler: error_handler_cb, - user_data: ptr::null_mut(), - }; - - let mut endpoint = ptr::null_mut(); - let status = unsafe { (self.api.ucp_ep_create)(self.worker, &ep_params, &mut endpoint) }; - - if status != UCS_OK { - return Err(RdmaError::context_init_failed(format!( - "ucp_ep_create failed: {} ({})", - self.api.status_string(status), status - ))); - } - - endpoints.insert(key.to_string(), endpoint); - Ok(endpoint) - } -} - -impl Drop for UcxContext { - fn drop(&mut self) { - info!("๐Ÿงน Cleaning up UCX context"); - - // Clean up endpoints - { - let mut endpoints = self.endpoints.lock(); - for (_, ep) in endpoints.drain() { - unsafe { (self.api.ucp_ep_destroy)(ep) }; - } - } - - // Clean up memory regions - { - let mut regions = self.memory_regions.lock(); - for (_, handle) in regions.drain() { - unsafe { (self.api.ucp_mem_unmap)(self.context, handle) }; - } - } - - // Clean up worker and context - unsafe { - (self.api.ucp_worker_destroy)(self.worker); - (self.api.ucp_cleanup)(self.context); - } - - info!("โœ… UCX context cleanup completed"); - } -} - -// UCX callback functions -extern "C" fn request_init_cb(_request: *mut c_void) { - // Request initialization callback -} - -extern "C" fn request_cleanup_cb(_request: *mut c_void) { - // Request cleanup callback -} - -extern "C" fn get_completion_cb(_request: *mut c_void, status: c_int, _user_data: *mut c_void) { - if status != UCS_OK { - error!("RDMA GET completion error: {}", status); - } -} - -extern "C" fn put_completion_cb(_request: *mut c_void, status: c_int, _user_data: *mut c_void) { - if status != UCS_OK { - error!("RDMA PUT completion error: {}", status); - } -} - -extern "C" fn error_handler_cb( - _arg: *mut c_void, - _ep: UcpEp, - status: c_int, -) { - error!("UCX endpoint error: {}", status); -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_ucx_api_loading() { - // This test will fail without UCX installed, which is expected - match UcxApi::load() { - Ok(api) => { - info!("UCX API loaded successfully"); - assert_eq!(api.status_string(UCS_OK), "Success"); - } - Err(_) => { - warn!("UCX library not found - expected in development environment"); - } - } - } - - #[tokio::test] - async fn test_ucx_context_mock() { - // This would test the mock implementation - // Real test requires UCX installation - } -} diff --git a/seaweedfs-rdma-sidecar/scripts/demo-e2e.sh b/seaweedfs-rdma-sidecar/scripts/demo-e2e.sh deleted file mode 100755 index 54a751e57..000000000 --- a/seaweedfs-rdma-sidecar/scripts/demo-e2e.sh +++ /dev/null @@ -1,314 +0,0 @@ -#!/bin/bash - -# SeaweedFS RDMA End-to-End Demo Script -# This script demonstrates the complete integration between SeaweedFS and the RDMA sidecar - -set -e - -# Configuration -RDMA_ENGINE_SOCKET="/tmp/rdma-engine.sock" -DEMO_SERVER_PORT=8080 -RUST_ENGINE_PID="" -DEMO_SERVER_PID="" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -print_header() { - echo -e "\n${PURPLE}===============================================${NC}" - echo -e "${PURPLE}$1${NC}" - echo -e "${PURPLE}===============================================${NC}\n" -} - -print_step() { - echo -e "${CYAN}๐Ÿ”ต $1${NC}" -} - -print_success() { - echo -e "${GREEN}โœ… $1${NC}" -} - -print_warning() { - echo -e "${YELLOW}โš ๏ธ $1${NC}" -} - -print_error() { - echo -e "${RED}โŒ $1${NC}" -} - -cleanup() { - print_header "CLEANUP" - - if [[ -n "$DEMO_SERVER_PID" ]]; then - print_step "Stopping demo server (PID: $DEMO_SERVER_PID)" - kill $DEMO_SERVER_PID 2>/dev/null || true - wait $DEMO_SERVER_PID 2>/dev/null || true - fi - - if [[ -n "$RUST_ENGINE_PID" ]]; then - print_step "Stopping Rust RDMA engine (PID: $RUST_ENGINE_PID)" - kill $RUST_ENGINE_PID 2>/dev/null || true - wait $RUST_ENGINE_PID 2>/dev/null || true - fi - - # Clean up socket - rm -f "$RDMA_ENGINE_SOCKET" - - print_success "Cleanup complete" -} - -# Set up cleanup on exit -trap cleanup EXIT - -build_components() { - print_header "BUILDING COMPONENTS" - - print_step "Building Go components..." - go build -o bin/demo-server ./cmd/demo-server - go build -o bin/test-rdma ./cmd/test-rdma - go build -o bin/sidecar ./cmd/sidecar - print_success "Go components built" - - print_step "Building Rust RDMA engine..." - cd rdma-engine - cargo build --release - cd .. - print_success "Rust RDMA engine built" -} - -start_rdma_engine() { - print_header "STARTING RDMA ENGINE" - - print_step "Starting Rust RDMA engine..." - ./rdma-engine/target/release/rdma-engine-server --debug & - RUST_ENGINE_PID=$! - - # Wait for engine to be ready - print_step "Waiting for RDMA engine to be ready..." - for i in {1..10}; do - if [[ -S "$RDMA_ENGINE_SOCKET" ]]; then - print_success "RDMA engine ready (PID: $RUST_ENGINE_PID)" - return 0 - fi - sleep 1 - done - - print_error "RDMA engine failed to start" - exit 1 -} - -start_demo_server() { - print_header "STARTING DEMO SERVER" - - print_step "Starting SeaweedFS RDMA demo server..." - ./bin/demo-server --port $DEMO_SERVER_PORT --rdma-socket "$RDMA_ENGINE_SOCKET" --enable-rdma --debug & - DEMO_SERVER_PID=$! - - # Wait for server to be ready - print_step "Waiting for demo server to be ready..." - for i in {1..10}; do - if curl -s "http://localhost:$DEMO_SERVER_PORT/health" > /dev/null 2>&1; then - print_success "Demo server ready (PID: $DEMO_SERVER_PID)" - return 0 - fi - sleep 1 - done - - print_error "Demo server failed to start" - exit 1 -} - -test_health_check() { - print_header "HEALTH CHECK TEST" - - print_step "Testing health endpoint..." - response=$(curl -s "http://localhost:$DEMO_SERVER_PORT/health") - - if echo "$response" | jq -e '.status == "healthy"' > /dev/null; then - print_success "Health check passed" - echo "$response" | jq '.' - else - print_error "Health check failed" - echo "$response" - exit 1 - fi -} - -test_capabilities() { - print_header "CAPABILITIES TEST" - - print_step "Testing capabilities endpoint..." - response=$(curl -s "http://localhost:$DEMO_SERVER_PORT/stats") - - if echo "$response" | jq -e '.enabled == true' > /dev/null; then - print_success "RDMA capabilities retrieved" - echo "$response" | jq '.' - else - print_warning "RDMA not enabled, but HTTP fallback available" - echo "$response" | jq '.' - fi -} - -test_needle_read() { - print_header "NEEDLE READ TEST" - - print_step "Testing RDMA needle read..." - response=$(curl -s "http://localhost:$DEMO_SERVER_PORT/read?volume=1&needle=12345&cookie=305419896&size=1024") - - if echo "$response" | jq -e '.success == true' > /dev/null; then - is_rdma=$(echo "$response" | jq -r '.is_rdma') - source=$(echo "$response" | jq -r '.source') - duration=$(echo "$response" | jq -r '.duration') - data_size=$(echo "$response" | jq -r '.data_size') - - if [[ "$is_rdma" == "true" ]]; then - print_success "RDMA fast path used! Duration: $duration, Size: $data_size bytes" - else - print_warning "HTTP fallback used. Duration: $duration, Size: $data_size bytes" - fi - - echo "$response" | jq '.' - else - print_error "Needle read failed" - echo "$response" - exit 1 - fi -} - -test_benchmark() { - print_header "PERFORMANCE BENCHMARK" - - print_step "Running performance benchmark..." - response=$(curl -s "http://localhost:$DEMO_SERVER_PORT/benchmark?iterations=5&size=2048") - - if echo "$response" | jq -e '.benchmark_results' > /dev/null; then - rdma_ops=$(echo "$response" | jq -r '.benchmark_results.rdma_ops') - http_ops=$(echo "$response" | jq -r '.benchmark_results.http_ops') - avg_latency=$(echo "$response" | jq -r '.benchmark_results.avg_latency') - throughput=$(echo "$response" | jq -r '.benchmark_results.throughput_mbps') - ops_per_sec=$(echo "$response" | jq -r '.benchmark_results.ops_per_sec') - - print_success "Benchmark completed:" - echo -e " ${BLUE}RDMA Operations:${NC} $rdma_ops" - echo -e " ${BLUE}HTTP Operations:${NC} $http_ops" - echo -e " ${BLUE}Average Latency:${NC} $avg_latency" - echo -e " ${BLUE}Throughput:${NC} $throughput MB/s" - echo -e " ${BLUE}Operations/sec:${NC} $ops_per_sec" - - echo -e "\n${BLUE}Full benchmark results:${NC}" - echo "$response" | jq '.benchmark_results' - else - print_error "Benchmark failed" - echo "$response" - exit 1 - fi -} - -test_direct_rdma() { - print_header "DIRECT RDMA ENGINE TEST" - - print_step "Testing direct RDMA engine communication..." - - echo "Testing ping..." - ./bin/test-rdma ping 2>/dev/null && print_success "Direct RDMA ping successful" || print_warning "Direct RDMA ping failed" - - echo -e "\nTesting capabilities..." - ./bin/test-rdma capabilities 2>/dev/null | head -15 && print_success "Direct RDMA capabilities successful" || print_warning "Direct RDMA capabilities failed" - - echo -e "\nTesting direct read..." - ./bin/test-rdma read --volume 1 --needle 12345 --size 1024 2>/dev/null > /dev/null && print_success "Direct RDMA read successful" || print_warning "Direct RDMA read failed" -} - -show_demo_urls() { - print_header "DEMO SERVER INFORMATION" - - echo -e "${GREEN}๐ŸŒ Demo server is running at: http://localhost:$DEMO_SERVER_PORT${NC}" - echo -e "${GREEN}๐Ÿ“ฑ Try these URLs:${NC}" - echo -e " ${BLUE}Home page:${NC} http://localhost:$DEMO_SERVER_PORT/" - echo -e " ${BLUE}Health check:${NC} http://localhost:$DEMO_SERVER_PORT/health" - echo -e " ${BLUE}Statistics:${NC} http://localhost:$DEMO_SERVER_PORT/stats" - echo -e " ${BLUE}Read needle:${NC} http://localhost:$DEMO_SERVER_PORT/read?volume=1&needle=12345&cookie=305419896&size=1024" - echo -e " ${BLUE}Benchmark:${NC} http://localhost:$DEMO_SERVER_PORT/benchmark?iterations=5&size=2048" - - echo -e "\n${GREEN}๐Ÿ“‹ Example curl commands:${NC}" - echo -e " ${CYAN}curl \"http://localhost:$DEMO_SERVER_PORT/health\" | jq '.'${NC}" - echo -e " ${CYAN}curl \"http://localhost:$DEMO_SERVER_PORT/read?volume=1&needle=12345&size=1024\" | jq '.'${NC}" - echo -e " ${CYAN}curl \"http://localhost:$DEMO_SERVER_PORT/benchmark?iterations=10\" | jq '.benchmark_results'${NC}" -} - -interactive_mode() { - print_header "INTERACTIVE MODE" - - show_demo_urls - - echo -e "\n${YELLOW}Press Enter to run automated tests, or Ctrl+C to exit and explore manually...${NC}" - read -r -} - -main() { - print_header "๐Ÿš€ SEAWEEDFS RDMA END-TO-END DEMO" - - echo -e "${GREEN}This demonstration shows:${NC}" - echo -e " โœ… Complete Go โ†” Rust IPC integration" - echo -e " โœ… SeaweedFS RDMA client with HTTP fallback" - echo -e " โœ… High-performance needle reads via RDMA" - echo -e " โœ… Performance benchmarking capabilities" - echo -e " โœ… Production-ready error handling and logging" - - # Check dependencies - if ! command -v jq &> /dev/null; then - print_error "jq is required for this demo. Please install it: brew install jq" - exit 1 - fi - - if ! command -v curl &> /dev/null; then - print_error "curl is required for this demo." - exit 1 - fi - - # Build and start components - build_components - start_rdma_engine - sleep 2 # Give engine time to fully initialize - start_demo_server - sleep 2 # Give server time to connect to engine - - # Show interactive information - interactive_mode - - # Run automated tests - test_health_check - test_capabilities - test_needle_read - test_benchmark - test_direct_rdma - - print_header "๐ŸŽ‰ END-TO-END DEMO COMPLETE!" - - echo -e "${GREEN}All tests passed successfully!${NC}" - echo -e "${BLUE}Key achievements demonstrated:${NC}" - echo -e " ๐Ÿš€ RDMA fast path working with mock operations" - echo -e " ๐Ÿ”„ Automatic HTTP fallback when RDMA unavailable" - echo -e " ๐Ÿ“Š Performance monitoring and benchmarking" - echo -e " ๐Ÿ›ก๏ธ Robust error handling and graceful degradation" - echo -e " ๐Ÿ”Œ Complete IPC protocol between Go and Rust" - echo -e " โšก Session management with proper cleanup" - - print_success "SeaweedFS RDMA integration is ready for hardware deployment!" - - # Keep server running for manual testing - echo -e "\n${YELLOW}Demo server will continue running for manual testing...${NC}" - echo -e "${YELLOW}Press Ctrl+C to shutdown.${NC}" - - # Wait for user interrupt - wait -} - -# Run the main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/demo-mount-rdma.sh b/seaweedfs-rdma-sidecar/scripts/demo-mount-rdma.sh deleted file mode 100755 index 890d6ccc0..000000000 --- a/seaweedfs-rdma-sidecar/scripts/demo-mount-rdma.sh +++ /dev/null @@ -1,249 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Configuration - assumes script is run from seaweedfs-rdma-sidecar directory -SEAWEEDFS_DIR="$(realpath ..)" -SIDECAR_DIR="$(pwd)" -MOUNT_POINT="/tmp/seaweedfs-rdma-mount" -FILER_ADDR="localhost:8888" -SIDECAR_ADDR="localhost:8081" - -# PIDs for cleanup -MASTER_PID="" -VOLUME_PID="" -FILER_PID="" -SIDECAR_PID="" -MOUNT_PID="" - -cleanup() { - echo -e "\n${YELLOW}๐Ÿงน Cleaning up processes...${NC}" - - # Unmount filesystem - if mountpoint -q "$MOUNT_POINT" 2>/dev/null; then - echo "๐Ÿ“ค Unmounting $MOUNT_POINT..." - fusermount -u "$MOUNT_POINT" 2>/dev/null || umount "$MOUNT_POINT" 2>/dev/null || true - sleep 1 - fi - - # Kill processes - for pid in $MOUNT_PID $SIDECAR_PID $FILER_PID $VOLUME_PID $MASTER_PID; do - if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then - echo "๐Ÿ”ช Killing process $pid..." - kill "$pid" 2>/dev/null || true - fi - done - - # Wait for processes to exit - sleep 2 - - # Force kill if necessary - for pid in $MOUNT_PID $SIDECAR_PID $FILER_PID $VOLUME_PID $MASTER_PID; do - if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then - echo "๐Ÿ’€ Force killing process $pid..." - kill -9 "$pid" 2>/dev/null || true - fi - done - - # Clean up mount point - if [[ -d "$MOUNT_POINT" ]]; then - rmdir "$MOUNT_POINT" 2>/dev/null || true - fi - - echo -e "${GREEN}โœ… Cleanup complete${NC}" -} - -trap cleanup EXIT - -wait_for_service() { - local name=$1 - local url=$2 - local max_attempts=30 - local attempt=1 - - echo -e "${BLUE}โณ Waiting for $name to be ready...${NC}" - - while [[ $attempt -le $max_attempts ]]; do - if curl -s "$url" >/dev/null 2>&1; then - echo -e "${GREEN}โœ… $name is ready${NC}" - return 0 - fi - echo " Attempt $attempt/$max_attempts..." - sleep 1 - ((attempt++)) - done - - echo -e "${RED}โŒ $name failed to start within $max_attempts seconds${NC}" - return 1 -} - -echo -e "${BLUE}๐Ÿš€ SEAWEEDFS RDMA MOUNT DEMONSTRATION${NC}" -echo "======================================" -echo "" -echo "This demo shows SeaweedFS mount with RDMA acceleration:" -echo " โ€ข Standard SeaweedFS cluster (master, volume, filer)" -echo " โ€ข RDMA sidecar for acceleration" -echo " โ€ข FUSE mount with RDMA fast path" -echo " โ€ข Performance comparison tests" -echo "" - -# Create mount point -echo -e "${BLUE}๐Ÿ“ Creating mount point: $MOUNT_POINT${NC}" -mkdir -p "$MOUNT_POINT" - -# Start SeaweedFS Master -echo -e "${BLUE}๐ŸŽฏ Starting SeaweedFS Master...${NC}" -cd "$SEAWEEDFS_DIR" -./weed master -port=9333 -mdir=/tmp/seaweedfs-master & -MASTER_PID=$! -wait_for_service "Master" "http://localhost:9333/cluster/status" - -# Start SeaweedFS Volume Server -echo -e "${BLUE}๐Ÿ’พ Starting SeaweedFS Volume Server...${NC}" -./weed volume -master=localhost:9333 -port=8080 -dir=/tmp/seaweedfs-volume & -VOLUME_PID=$! -wait_for_service "Volume Server" "http://localhost:8080/status" - -# Start SeaweedFS Filer -echo -e "${BLUE}๐Ÿ“‚ Starting SeaweedFS Filer...${NC}" -./weed filer -master=localhost:9333 -port=8888 & -FILER_PID=$! -wait_for_service "Filer" "http://localhost:8888/" - -# Start RDMA Sidecar -echo -e "${BLUE}โšก Starting RDMA Sidecar...${NC}" -cd "$SIDECAR_DIR" -./bin/demo-server --port 8081 --rdma-socket /tmp/rdma-engine.sock --volume-server-url http://localhost:8080 --enable-rdma --debug & -SIDECAR_PID=$! -wait_for_service "RDMA Sidecar" "http://localhost:8081/health" - -# Check RDMA capabilities -echo -e "${BLUE}๐Ÿ” Checking RDMA capabilities...${NC}" -curl -s "http://localhost:8081/stats" | jq . || curl -s "http://localhost:8081/stats" - -echo "" -echo -e "${BLUE}๐Ÿ—‚๏ธ Mounting SeaweedFS with RDMA acceleration...${NC}" - -# Mount with RDMA acceleration -cd "$SEAWEEDFS_DIR" -./weed mount \ - -filer="$FILER_ADDR" \ - -dir="$MOUNT_POINT" \ - -rdma.enabled=true \ - -rdma.sidecar="$SIDECAR_ADDR" \ - -rdma.fallback=true \ - -rdma.maxConcurrent=64 \ - -rdma.timeoutMs=5000 \ - -debug=true & -MOUNT_PID=$! - -# Wait for mount to be ready -echo -e "${BLUE}โณ Waiting for mount to be ready...${NC}" -sleep 5 - -# Check if mount is successful -if ! mountpoint -q "$MOUNT_POINT"; then - echo -e "${RED}โŒ Mount failed${NC}" - exit 1 -fi - -echo -e "${GREEN}โœ… SeaweedFS mounted successfully with RDMA acceleration!${NC}" -echo "" - -# Demonstrate RDMA-accelerated operations -echo -e "${BLUE}๐Ÿงช TESTING RDMA-ACCELERATED FILE OPERATIONS${NC}" -echo "==============================================" - -# Create test files -echo -e "${BLUE}๐Ÿ“ Creating test files...${NC}" -echo "Hello, RDMA World!" > "$MOUNT_POINT/test1.txt" -echo "This file will be read via RDMA acceleration!" > "$MOUNT_POINT/test2.txt" - -# Create a larger test file -echo -e "${BLUE}๐Ÿ“ Creating larger test file (1MB)...${NC}" -dd if=/dev/zero of="$MOUNT_POINT/large_test.dat" bs=1024 count=1024 2>/dev/null - -echo -e "${GREEN}โœ… Test files created${NC}" -echo "" - -# Test file reads -echo -e "${BLUE}๐Ÿ“– Testing file reads (should use RDMA fast path)...${NC}" -echo "" - -echo "๐Ÿ“„ Reading test1.txt:" -cat "$MOUNT_POINT/test1.txt" -echo "" - -echo "๐Ÿ“„ Reading test2.txt:" -cat "$MOUNT_POINT/test2.txt" -echo "" - -echo "๐Ÿ“„ Reading first 100 bytes of large file:" -head -c 100 "$MOUNT_POINT/large_test.dat" | hexdump -C | head -5 -echo "" - -# Performance test -echo -e "${BLUE}๐Ÿ PERFORMANCE COMPARISON${NC}" -echo "=========================" - -echo "๐Ÿ”ฅ Testing read performance with RDMA acceleration..." -time_start=$(date +%s%N) -for i in {1..10}; do - cat "$MOUNT_POINT/large_test.dat" > /dev/null -done -time_end=$(date +%s%N) -rdma_time=$((($time_end - $time_start) / 1000000)) # Convert to milliseconds - -echo "โœ… RDMA-accelerated reads: 10 x 1MB file = ${rdma_time}ms total" -echo "" - -# Check RDMA statistics -echo -e "${BLUE}๐Ÿ“Š RDMA Statistics:${NC}" -curl -s "http://localhost:8081/stats" | jq . 2>/dev/null || curl -s "http://localhost:8081/stats" -echo "" - -# List files -echo -e "${BLUE}๐Ÿ“‹ Files in mounted filesystem:${NC}" -ls -la "$MOUNT_POINT/" -echo "" - -# Interactive mode -echo -e "${BLUE}๐ŸŽฎ INTERACTIVE MODE${NC}" -echo "==================" -echo "" -echo "The SeaweedFS filesystem is now mounted at: $MOUNT_POINT" -echo "RDMA acceleration is active for all read operations!" -echo "" -echo "Try these commands:" -echo " ls $MOUNT_POINT/" -echo " cat $MOUNT_POINT/test1.txt" -echo " echo 'New content' > $MOUNT_POINT/new_file.txt" -echo " cat $MOUNT_POINT/new_file.txt" -echo "" -echo "Monitor RDMA stats: curl http://localhost:8081/stats | jq" -echo "Check mount status: mount | grep seaweedfs" -echo "" -echo -e "${YELLOW}Press Ctrl+C to stop the demo and cleanup${NC}" - -# Keep running until interrupted -while true; do - sleep 5 - - # Check if mount is still active - if ! mountpoint -q "$MOUNT_POINT"; then - echo -e "${RED}โŒ Mount point lost, exiting...${NC}" - break - fi - - # Show periodic stats - echo -e "${BLUE}๐Ÿ“Š Current RDMA stats ($(date)):${NC}" - curl -s "http://localhost:8081/stats" | jq '.rdma_enabled, .total_reads, .rdma_reads, .http_fallbacks' 2>/dev/null || echo "Stats unavailable" - echo "" -done diff --git a/seaweedfs-rdma-sidecar/scripts/mount-health-check.sh b/seaweedfs-rdma-sidecar/scripts/mount-health-check.sh deleted file mode 100755 index 4565cc617..000000000 --- a/seaweedfs-rdma-sidecar/scripts/mount-health-check.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} - -# Check if mount point exists and is mounted -if [[ ! -d "$MOUNT_POINT" ]]; then - echo "Mount point $MOUNT_POINT does not exist" - exit 1 -fi - -if ! mountpoint -q "$MOUNT_POINT"; then - echo "Mount point $MOUNT_POINT is not mounted" - exit 1 -fi - -# Try to list the mount point -if ! ls "$MOUNT_POINT" >/dev/null 2>&1; then - echo "Cannot list mount point $MOUNT_POINT" - exit 1 -fi - -echo "Mount point $MOUNT_POINT is healthy" -exit 0 diff --git a/seaweedfs-rdma-sidecar/scripts/mount-helper.sh b/seaweedfs-rdma-sidecar/scripts/mount-helper.sh deleted file mode 100755 index 4159dd180..000000000 --- a/seaweedfs-rdma-sidecar/scripts/mount-helper.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Configuration from environment variables -FILER_ADDR=${FILER_ADDR:-"seaweedfs-filer:8888"} -RDMA_SIDECAR_ADDR=${RDMA_SIDECAR_ADDR:-"rdma-sidecar:8081"} -MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} -RDMA_ENABLED=${RDMA_ENABLED:-"true"} -RDMA_FALLBACK=${RDMA_FALLBACK:-"true"} -RDMA_MAX_CONCURRENT=${RDMA_MAX_CONCURRENT:-"64"} -RDMA_TIMEOUT_MS=${RDMA_TIMEOUT_MS:-"5000"} -DEBUG=${DEBUG:-"false"} - -echo -e "${BLUE}๐Ÿš€ SeaweedFS RDMA Mount Helper${NC}" -echo "================================" -echo "Filer Address: $FILER_ADDR" -echo "RDMA Sidecar: $RDMA_SIDECAR_ADDR" -echo "Mount Point: $MOUNT_POINT" -echo "RDMA Enabled: $RDMA_ENABLED" -echo "RDMA Fallback: $RDMA_FALLBACK" -echo "Debug Mode: $DEBUG" -echo "" - -# Function to wait for service -wait_for_service() { - local name=$1 - local url=$2 - local max_attempts=30 - local attempt=1 - - echo -e "${BLUE}โณ Waiting for $name to be ready...${NC}" - - while [[ $attempt -le $max_attempts ]]; do - if curl -s "$url" >/dev/null 2>&1; then - echo -e "${GREEN}โœ… $name is ready${NC}" - return 0 - fi - echo " Attempt $attempt/$max_attempts..." - sleep 2 - ((attempt++)) - done - - echo -e "${RED}โŒ $name failed to be ready within $max_attempts attempts${NC}" - return 1 -} - -# Function to check RDMA sidecar capabilities -check_rdma_capabilities() { - echo -e "${BLUE}๐Ÿ” Checking RDMA capabilities...${NC}" - - local response - if response=$(curl -s "http://$RDMA_SIDECAR_ADDR/stats" 2>/dev/null); then - echo "RDMA Sidecar Stats:" - echo "$response" | jq . 2>/dev/null || echo "$response" - echo "" - - # Check if RDMA is actually enabled - if echo "$response" | grep -q '"rdma_enabled":true'; then - echo -e "${GREEN}โœ… RDMA is enabled and ready${NC}" - return 0 - else - echo -e "${YELLOW}โš ๏ธ RDMA sidecar is running but RDMA is not enabled${NC}" - if [[ "$RDMA_FALLBACK" == "true" ]]; then - echo -e "${YELLOW} Will use HTTP fallback${NC}" - return 0 - else - return 1 - fi - fi - else - echo -e "${RED}โŒ Failed to get RDMA sidecar stats${NC}" - if [[ "$RDMA_FALLBACK" == "true" ]]; then - echo -e "${YELLOW} Will use HTTP fallback${NC}" - return 0 - else - return 1 - fi - fi -} - -# Function to cleanup on exit -cleanup() { - echo -e "\n${YELLOW}๐Ÿงน Cleaning up...${NC}" - - # Unmount if mounted - if mountpoint -q "$MOUNT_POINT" 2>/dev/null; then - echo "๐Ÿ“ค Unmounting $MOUNT_POINT..." - fusermount3 -u "$MOUNT_POINT" 2>/dev/null || umount "$MOUNT_POINT" 2>/dev/null || true - sleep 2 - fi - - echo -e "${GREEN}โœ… Cleanup complete${NC}" -} - -trap cleanup EXIT INT TERM - -# Wait for required services -echo -e "${BLUE}๐Ÿ”„ Waiting for required services...${NC}" -wait_for_service "Filer" "http://$FILER_ADDR/" - -if [[ "$RDMA_ENABLED" == "true" ]]; then - wait_for_service "RDMA Sidecar" "http://$RDMA_SIDECAR_ADDR/health" - check_rdma_capabilities -fi - -# Create mount point if it doesn't exist -echo -e "${BLUE}๐Ÿ“ Preparing mount point...${NC}" -mkdir -p "$MOUNT_POINT" - -# Check if already mounted -if mountpoint -q "$MOUNT_POINT"; then - echo -e "${YELLOW}โš ๏ธ $MOUNT_POINT is already mounted, unmounting first...${NC}" - fusermount3 -u "$MOUNT_POINT" 2>/dev/null || umount "$MOUNT_POINT" 2>/dev/null || true - sleep 2 -fi - -# Build mount command -MOUNT_CMD="/usr/local/bin/weed mount" -MOUNT_CMD="$MOUNT_CMD -filer=$FILER_ADDR" -MOUNT_CMD="$MOUNT_CMD -dir=$MOUNT_POINT" -MOUNT_CMD="$MOUNT_CMD -allowOthers=true" - -# Add RDMA options if enabled -if [[ "$RDMA_ENABLED" == "true" ]]; then - MOUNT_CMD="$MOUNT_CMD -rdma.enabled=true" - MOUNT_CMD="$MOUNT_CMD -rdma.sidecar=$RDMA_SIDECAR_ADDR" - MOUNT_CMD="$MOUNT_CMD -rdma.fallback=$RDMA_FALLBACK" - MOUNT_CMD="$MOUNT_CMD -rdma.maxConcurrent=$RDMA_MAX_CONCURRENT" - MOUNT_CMD="$MOUNT_CMD -rdma.timeoutMs=$RDMA_TIMEOUT_MS" -fi - -# Add debug options if enabled -if [[ "$DEBUG" == "true" ]]; then - MOUNT_CMD="$MOUNT_CMD -debug=true -v=2" -fi - -echo -e "${BLUE}๐Ÿ—‚๏ธ Starting SeaweedFS mount...${NC}" -echo "Command: $MOUNT_CMD" -echo "" - -# Execute mount command -exec $MOUNT_CMD diff --git a/seaweedfs-rdma-sidecar/scripts/performance-benchmark.sh b/seaweedfs-rdma-sidecar/scripts/performance-benchmark.sh deleted file mode 100755 index 907cf5a7a..000000000 --- a/seaweedfs-rdma-sidecar/scripts/performance-benchmark.sh +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash - -# Performance Benchmark Script -# Tests the revolutionary zero-copy + connection pooling optimizations - -set -e - -echo "๐Ÿš€ SeaweedFS RDMA Performance Benchmark" -echo "Testing Zero-Copy Page Cache + Connection Pooling Optimizations" -echo "==============================================================" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -# Test configuration -SIDECAR_URL="http://localhost:8081" -TEST_VOLUME=1 -TEST_NEEDLE=1 -TEST_COOKIE=1 -ITERATIONS=10 - -# File sizes to test (representing different optimization thresholds) -declare -a SIZES=( - "4096" # 4KB - Small file (below zero-copy threshold) - "32768" # 32KB - Medium file (below zero-copy threshold) - "65536" # 64KB - Zero-copy threshold - "262144" # 256KB - Medium zero-copy file - "1048576" # 1MB - Large zero-copy file - "10485760" # 10MB - Very large zero-copy file -) - -declare -a SIZE_NAMES=( - "4KB" - "32KB" - "64KB" - "256KB" - "1MB" - "10MB" -) - -# Function to check if sidecar is ready -check_sidecar() { - echo -n "Waiting for RDMA sidecar to be ready..." - for i in {1..30}; do - if curl -s "$SIDECAR_URL/health" > /dev/null 2>&1; then - echo -e " ${GREEN}โœ“ Ready${NC}" - return 0 - fi - echo -n "." - sleep 2 - done - echo -e " ${RED}โœ— Failed${NC}" - return 1 -} - -# Function to perform benchmark for a specific size -benchmark_size() { - local size=$1 - local size_name=$2 - - echo -e "\n${CYAN}๐Ÿ“Š Testing ${size_name} files (${size} bytes)${NC}" - echo "----------------------------------------" - - local total_time=0 - local rdma_count=0 - local zerocopy_count=0 - local pooled_count=0 - - for i in $(seq 1 $ITERATIONS); do - echo -n " Iteration $i/$ITERATIONS: " - - # Make request with volume_server parameter - local start_time=$(date +%s%N) - local response=$(curl -s "$SIDECAR_URL/read?volume=$TEST_VOLUME&needle=$TEST_NEEDLE&cookie=$TEST_COOKIE&size=$size&volume_server=http://seaweedfs-volume:8080") - local end_time=$(date +%s%N) - - # Calculate duration in milliseconds - local duration_ns=$((end_time - start_time)) - local duration_ms=$((duration_ns / 1000000)) - - total_time=$((total_time + duration_ms)) - - # Parse response to check optimization flags - local is_rdma=$(echo "$response" | jq -r '.is_rdma // false' 2>/dev/null || echo "false") - local source=$(echo "$response" | jq -r '.source // "unknown"' 2>/dev/null || echo "unknown") - local use_temp_file=$(echo "$response" | jq -r '.use_temp_file // false' 2>/dev/null || echo "false") - - # Count optimization usage - if [[ "$is_rdma" == "true" ]]; then - rdma_count=$((rdma_count + 1)) - fi - - if [[ "$source" == *"zerocopy"* ]] || [[ "$use_temp_file" == "true" ]]; then - zerocopy_count=$((zerocopy_count + 1)) - fi - - if [[ "$source" == *"pooled"* ]]; then - pooled_count=$((pooled_count + 1)) - fi - - # Display result with color coding - if [[ "$source" == "rdma-zerocopy" ]]; then - echo -e "${GREEN}${duration_ms}ms (RDMA+ZeroCopy)${NC}" - elif [[ "$is_rdma" == "true" ]]; then - echo -e "${YELLOW}${duration_ms}ms (RDMA)${NC}" - else - echo -e "${RED}${duration_ms}ms (HTTP)${NC}" - fi - done - - # Calculate statistics - local avg_time=$((total_time / ITERATIONS)) - local rdma_percentage=$((rdma_count * 100 / ITERATIONS)) - local zerocopy_percentage=$((zerocopy_count * 100 / ITERATIONS)) - local pooled_percentage=$((pooled_count * 100 / ITERATIONS)) - - echo -e "\n${PURPLE}๐Ÿ“ˆ Results for ${size_name}:${NC}" - echo " Average latency: ${avg_time}ms" - echo " RDMA usage: ${rdma_percentage}%" - echo " Zero-copy usage: ${zerocopy_percentage}%" - echo " Connection pooling: ${pooled_percentage}%" - - # Performance assessment - if [[ $zerocopy_percentage -gt 80 ]]; then - echo -e " ${GREEN}๐Ÿ”ฅ REVOLUTIONARY: Zero-copy optimization active!${NC}" - elif [[ $rdma_percentage -gt 80 ]]; then - echo -e " ${YELLOW}โšก EXCELLENT: RDMA acceleration active${NC}" - else - echo -e " ${RED}โš ๏ธ WARNING: Falling back to HTTP${NC}" - fi - - # Store results for comparison - echo "$size_name,$avg_time,$rdma_percentage,$zerocopy_percentage,$pooled_percentage" >> /tmp/benchmark_results.csv -} - -# Function to display final performance analysis -performance_analysis() { - echo -e "\n${BLUE}๐ŸŽฏ PERFORMANCE ANALYSIS${NC}" - echo "========================================" - - if [[ -f /tmp/benchmark_results.csv ]]; then - echo -e "\n${CYAN}Summary Results:${NC}" - echo "Size | Avg Latency | RDMA % | Zero-Copy % | Pooled %" - echo "---------|-------------|--------|-------------|----------" - - while IFS=',' read -r size_name avg_time rdma_pct zerocopy_pct pooled_pct; do - printf "%-8s | %-11s | %-6s | %-11s | %-8s\n" "$size_name" "${avg_time}ms" "${rdma_pct}%" "${zerocopy_pct}%" "${pooled_pct}%" - done < /tmp/benchmark_results.csv - fi - - echo -e "\n${GREEN}๐Ÿš€ OPTIMIZATION IMPACT:${NC}" - echo "โ€ข Zero-Copy Page Cache: Eliminates 4/5 memory copies" - echo "โ€ข Connection Pooling: Eliminates 100ms RDMA setup cost" - echo "โ€ข Combined Effect: Up to 118x performance improvement!" - - echo -e "\n${PURPLE}๐Ÿ“Š Expected vs Actual Performance:${NC}" - echo "โ€ข Small files (4-32KB): Expected 50x faster copies" - echo "โ€ข Medium files (64-256KB): Expected 25x faster copies + instant connection" - echo "โ€ข Large files (1MB+): Expected 100x faster copies + instant connection" - - # Check if connection pooling is working - echo -e "\n${CYAN}๐Ÿ”Œ Connection Pooling Analysis:${NC}" - local stats_response=$(curl -s "$SIDECAR_URL/stats" 2>/dev/null || echo "{}") - local total_requests=$(echo "$stats_response" | jq -r '.total_requests // 0' 2>/dev/null || echo "0") - - if [[ "$total_requests" -gt 0 ]]; then - echo "โœ… Connection pooling is functional" - echo " Total requests processed: $total_requests" - else - echo "โš ๏ธ Unable to retrieve connection pool statistics" - fi - - rm -f /tmp/benchmark_results.csv -} - -# Main execution -main() { - echo -e "\n${YELLOW}๐Ÿ”ง Initializing benchmark...${NC}" - - # Check if sidecar is ready - if ! check_sidecar; then - echo -e "${RED}โŒ RDMA sidecar is not ready. Please start the Docker environment first.${NC}" - echo "Run: cd /path/to/seaweedfs-rdma-sidecar && docker compose -f docker-compose.mount-rdma.yml up -d" - exit 1 - fi - - # Initialize results file - rm -f /tmp/benchmark_results.csv - - # Run benchmarks for each file size - for i in "${!SIZES[@]}"; do - benchmark_size "${SIZES[$i]}" "${SIZE_NAMES[$i]}" - done - - # Display final analysis - performance_analysis - - echo -e "\n${GREEN}๐ŸŽ‰ Benchmark completed!${NC}" -} - -# Run the benchmark -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/run-integration-tests.sh b/seaweedfs-rdma-sidecar/scripts/run-integration-tests.sh deleted file mode 100755 index a9e5bd644..000000000 --- a/seaweedfs-rdma-sidecar/scripts/run-integration-tests.sh +++ /dev/null @@ -1,288 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Configuration -MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} -FILER_ADDR=${FILER_ADDR:-"seaweedfs-filer:8888"} -RDMA_SIDECAR_ADDR=${RDMA_SIDECAR_ADDR:-"rdma-sidecar:8081"} -TEST_RESULTS_DIR=${TEST_RESULTS_DIR:-"/test-results"} - -# Test counters -TOTAL_TESTS=0 -PASSED_TESTS=0 -FAILED_TESTS=0 - -# Create results directory -mkdir -p "$TEST_RESULTS_DIR" - -# Log file -LOG_FILE="$TEST_RESULTS_DIR/integration-test.log" -exec > >(tee -a "$LOG_FILE") -exec 2>&1 - -echo -e "${BLUE}๐Ÿงช SEAWEEDFS RDMA MOUNT INTEGRATION TESTS${NC}" -echo "==========================================" -echo "Mount Point: $MOUNT_POINT" -echo "Filer Address: $FILER_ADDR" -echo "RDMA Sidecar: $RDMA_SIDECAR_ADDR" -echo "Results Directory: $TEST_RESULTS_DIR" -echo "Log File: $LOG_FILE" -echo "" - -# Function to run a test -run_test() { - local test_name=$1 - local test_command=$2 - - echo -e "${BLUE}๐Ÿ”ฌ Running test: $test_name${NC}" - ((TOTAL_TESTS++)) - - if eval "$test_command"; then - echo -e "${GREEN}โœ… PASSED: $test_name${NC}" - ((PASSED_TESTS++)) - echo "PASS" > "$TEST_RESULTS_DIR/${test_name}.result" - else - echo -e "${RED}โŒ FAILED: $test_name${NC}" - ((FAILED_TESTS++)) - echo "FAIL" > "$TEST_RESULTS_DIR/${test_name}.result" - fi - echo "" -} - -# Function to wait for mount to be ready -wait_for_mount() { - local max_attempts=30 - local attempt=1 - - echo -e "${BLUE}โณ Waiting for mount to be ready...${NC}" - - while [[ $attempt -le $max_attempts ]]; do - if mountpoint -q "$MOUNT_POINT" 2>/dev/null && ls "$MOUNT_POINT" >/dev/null 2>&1; then - echo -e "${GREEN}โœ… Mount is ready${NC}" - return 0 - fi - echo " Attempt $attempt/$max_attempts..." - sleep 2 - ((attempt++)) - done - - echo -e "${RED}โŒ Mount failed to be ready${NC}" - return 1 -} - -# Function to check RDMA sidecar -check_rdma_sidecar() { - echo -e "${BLUE}๐Ÿ” Checking RDMA sidecar status...${NC}" - - local response - if response=$(curl -s "http://$RDMA_SIDECAR_ADDR/health" 2>/dev/null); then - echo "RDMA Sidecar Health: $response" - return 0 - else - echo -e "${RED}โŒ RDMA sidecar is not responding${NC}" - return 1 - fi -} - -# Test 1: Mount Point Accessibility -test_mount_accessibility() { - mountpoint -q "$MOUNT_POINT" && ls "$MOUNT_POINT" >/dev/null -} - -# Test 2: Basic File Operations -test_basic_file_operations() { - local test_file="$MOUNT_POINT/test_basic_ops.txt" - local test_content="Hello, RDMA World! $(date)" - - # Write test - echo "$test_content" > "$test_file" || return 1 - - # Read test - local read_content - read_content=$(cat "$test_file") || return 1 - - # Verify content - [[ "$read_content" == "$test_content" ]] || return 1 - - # Cleanup - rm -f "$test_file" - - return 0 -} - -# Test 3: Large File Operations -test_large_file_operations() { - local test_file="$MOUNT_POINT/test_large_file.dat" - local size_mb=10 - - # Create large file - dd if=/dev/zero of="$test_file" bs=1M count=$size_mb 2>/dev/null || return 1 - - # Verify size - local actual_size - actual_size=$(stat -c%s "$test_file" 2>/dev/null) || return 1 - local expected_size=$((size_mb * 1024 * 1024)) - - [[ "$actual_size" -eq "$expected_size" ]] || return 1 - - # Read test - dd if="$test_file" of=/dev/null bs=1M 2>/dev/null || return 1 - - # Cleanup - rm -f "$test_file" - - return 0 -} - -# Test 4: Directory Operations -test_directory_operations() { - local test_dir="$MOUNT_POINT/test_directory" - local test_file="$test_dir/test_file.txt" - - # Create directory - mkdir -p "$test_dir" || return 1 - - # Create file in directory - echo "Directory test" > "$test_file" || return 1 - - # List directory - ls "$test_dir" | grep -q "test_file.txt" || return 1 - - # Read file - grep -q "Directory test" "$test_file" || return 1 - - # Cleanup - rm -rf "$test_dir" - - return 0 -} - -# Test 5: Multiple File Operations -test_multiple_files() { - local test_dir="$MOUNT_POINT/test_multiple" - local num_files=20 - - mkdir -p "$test_dir" || return 1 - - # Create multiple files - for i in $(seq 1 $num_files); do - echo "File $i content" > "$test_dir/file_$i.txt" || return 1 - done - - # Verify all files exist and have correct content - for i in $(seq 1 $num_files); do - [[ -f "$test_dir/file_$i.txt" ]] || return 1 - grep -q "File $i content" "$test_dir/file_$i.txt" || return 1 - done - - # List files - local file_count - file_count=$(ls "$test_dir" | wc -l) || return 1 - [[ "$file_count" -eq "$num_files" ]] || return 1 - - # Cleanup - rm -rf "$test_dir" - - return 0 -} - -# Test 6: RDMA Statistics -test_rdma_statistics() { - local stats_response - stats_response=$(curl -s "http://$RDMA_SIDECAR_ADDR/stats" 2>/dev/null) || return 1 - - # Check if response contains expected fields - echo "$stats_response" | jq -e '.rdma_enabled' >/dev/null || return 1 - echo "$stats_response" | jq -e '.total_reads' >/dev/null || return 1 - - return 0 -} - -# Test 7: Performance Baseline -test_performance_baseline() { - local test_file="$MOUNT_POINT/performance_test.dat" - local size_mb=50 - - # Write performance test - local write_start write_end write_time - write_start=$(date +%s%N) - dd if=/dev/zero of="$test_file" bs=1M count=$size_mb 2>/dev/null || return 1 - write_end=$(date +%s%N) - write_time=$(((write_end - write_start) / 1000000)) # Convert to milliseconds - - # Read performance test - local read_start read_end read_time - read_start=$(date +%s%N) - dd if="$test_file" of=/dev/null bs=1M 2>/dev/null || return 1 - read_end=$(date +%s%N) - read_time=$(((read_end - read_start) / 1000000)) # Convert to milliseconds - - # Log performance metrics - echo "Performance Metrics:" > "$TEST_RESULTS_DIR/performance.txt" - echo "Write Time: ${write_time}ms for ${size_mb}MB" >> "$TEST_RESULTS_DIR/performance.txt" - echo "Read Time: ${read_time}ms for ${size_mb}MB" >> "$TEST_RESULTS_DIR/performance.txt" - echo "Write Throughput: $(bc <<< "scale=2; $size_mb * 1000 / $write_time") MB/s" >> "$TEST_RESULTS_DIR/performance.txt" - echo "Read Throughput: $(bc <<< "scale=2; $size_mb * 1000 / $read_time") MB/s" >> "$TEST_RESULTS_DIR/performance.txt" - - # Cleanup - rm -f "$test_file" - - # Performance test always passes (it's just for metrics) - return 0 -} - -# Main test execution -main() { - echo -e "${BLUE}๐Ÿš€ Starting integration tests...${NC}" - echo "" - - # Wait for mount to be ready - if ! wait_for_mount; then - echo -e "${RED}โŒ Mount is not ready, aborting tests${NC}" - exit 1 - fi - - # Check RDMA sidecar - check_rdma_sidecar || echo -e "${YELLOW}โš ๏ธ RDMA sidecar check failed, continuing with tests${NC}" - - echo "" - echo -e "${BLUE}๐Ÿ“‹ Running test suite...${NC}" - echo "" - - # Run all tests - run_test "mount_accessibility" "test_mount_accessibility" - run_test "basic_file_operations" "test_basic_file_operations" - run_test "large_file_operations" "test_large_file_operations" - run_test "directory_operations" "test_directory_operations" - run_test "multiple_files" "test_multiple_files" - run_test "rdma_statistics" "test_rdma_statistics" - run_test "performance_baseline" "test_performance_baseline" - - # Generate test summary - echo -e "${BLUE}๐Ÿ“Š TEST SUMMARY${NC}" - echo "===============" - echo "Total Tests: $TOTAL_TESTS" - echo -e "Passed: ${GREEN}$PASSED_TESTS${NC}" - echo -e "Failed: ${RED}$FAILED_TESTS${NC}" - - if [[ $FAILED_TESTS -eq 0 ]]; then - echo -e "${GREEN}๐ŸŽ‰ ALL TESTS PASSED!${NC}" - echo "SUCCESS" > "$TEST_RESULTS_DIR/overall.result" - exit 0 - else - echo -e "${RED}๐Ÿ’ฅ SOME TESTS FAILED!${NC}" - echo "FAILURE" > "$TEST_RESULTS_DIR/overall.result" - exit 1 - fi -} - -# Run main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/run-mount-rdma-tests.sh b/seaweedfs-rdma-sidecar/scripts/run-mount-rdma-tests.sh deleted file mode 100755 index e4237a5a2..000000000 --- a/seaweedfs-rdma-sidecar/scripts/run-mount-rdma-tests.sh +++ /dev/null @@ -1,335 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Configuration -COMPOSE_FILE="docker-compose.mount-rdma.yml" -PROJECT_NAME="seaweedfs-rdma-mount" - -# Function to show usage -show_usage() { - echo -e "${BLUE}๐Ÿš€ SeaweedFS RDMA Mount Test Runner${NC}" - echo "====================================" - echo "" - echo "Usage: $0 [COMMAND] [OPTIONS]" - echo "" - echo "Commands:" - echo " start Start the RDMA mount environment" - echo " stop Stop and cleanup the environment" - echo " restart Restart the environment" - echo " status Show status of all services" - echo " logs [service] Show logs for all services or specific service" - echo " test Run integration tests" - echo " perf Run performance tests" - echo " shell Open shell in mount container" - echo " cleanup Full cleanup including volumes" - echo "" - echo "Services:" - echo " seaweedfs-master SeaweedFS master server" - echo " seaweedfs-volume SeaweedFS volume server" - echo " seaweedfs-filer SeaweedFS filer server" - echo " rdma-engine RDMA engine (Rust)" - echo " rdma-sidecar RDMA sidecar (Go)" - echo " seaweedfs-mount SeaweedFS mount with RDMA" - echo "" - echo "Examples:" - echo " $0 start # Start all services" - echo " $0 logs seaweedfs-mount # Show mount logs" - echo " $0 test # Run integration tests" - echo " $0 perf # Run performance tests" - echo " $0 shell # Open shell in mount container" -} - -# Function to check if Docker Compose is available -check_docker_compose() { - if ! command -v docker-compose >/dev/null 2>&1 && ! docker compose version >/dev/null 2>&1; then - echo -e "${RED}โŒ Docker Compose is not available${NC}" - echo "Please install Docker Compose to continue" - exit 1 - fi - - # Use docker compose if available, otherwise docker-compose - if docker compose version >/dev/null 2>&1; then - DOCKER_COMPOSE="docker compose" - else - DOCKER_COMPOSE="docker-compose" - fi -} - -# Function to build required images -build_images() { - echo -e "${BLUE}๐Ÿ”จ Building required Docker images...${NC}" - - # Build SeaweedFS binary first - echo "Building SeaweedFS binary..." - cd .. - make - cd seaweedfs-rdma-sidecar - - # Copy binary for Docker builds - mkdir -p bin - if [[ -f "../weed" ]]; then - cp ../weed bin/ - elif [[ -f "../bin/weed" ]]; then - cp ../bin/weed bin/ - elif [[ -f "../build/weed" ]]; then - cp ../build/weed bin/ - else - echo "Error: Cannot find weed binary" - find .. -name "weed" -type f - exit 1 - fi - - # Build RDMA sidecar - echo "Building RDMA sidecar..." - go build -o bin/demo-server cmd/sidecar/main.go - - # Build Docker images - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" build - - echo -e "${GREEN}โœ… Images built successfully${NC}" -} - -# Function to start services -start_services() { - echo -e "${BLUE}๐Ÿš€ Starting SeaweedFS RDMA Mount environment...${NC}" - - # Build images if needed - if [[ ! -f "bin/weed" ]] || [[ ! -f "bin/demo-server" ]]; then - build_images - fi - - # Start services - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" up -d - - echo -e "${GREEN}โœ… Services started${NC}" - echo "" - echo "Services are starting up. Use '$0 status' to check their status." - echo "Use '$0 logs' to see the logs." -} - -# Function to stop services -stop_services() { - echo -e "${BLUE}๐Ÿ›‘ Stopping SeaweedFS RDMA Mount environment...${NC}" - - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" down - - echo -e "${GREEN}โœ… Services stopped${NC}" -} - -# Function to restart services -restart_services() { - echo -e "${BLUE}๐Ÿ”„ Restarting SeaweedFS RDMA Mount environment...${NC}" - - stop_services - sleep 2 - start_services -} - -# Function to show status -show_status() { - echo -e "${BLUE}๐Ÿ“Š Service Status${NC}" - echo "================" - - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" ps - - echo "" - echo -e "${BLUE}๐Ÿ” Health Checks${NC}" - echo "===============" - - # Check individual services - check_service_health "SeaweedFS Master" "http://localhost:9333/cluster/status" - check_service_health "SeaweedFS Volume" "http://localhost:8080/status" - check_service_health "SeaweedFS Filer" "http://localhost:8888/" - check_service_health "RDMA Sidecar" "http://localhost:8081/health" - - # Check mount status - echo -n "SeaweedFS Mount: " - if docker exec "${PROJECT_NAME}-seaweedfs-mount-1" mountpoint -q /mnt/seaweedfs 2>/dev/null; then - echo -e "${GREEN}โœ… Mounted${NC}" - else - echo -e "${RED}โŒ Not mounted${NC}" - fi -} - -# Function to check service health -check_service_health() { - local service_name=$1 - local health_url=$2 - - echo -n "$service_name: " - if curl -s "$health_url" >/dev/null 2>&1; then - echo -e "${GREEN}โœ… Healthy${NC}" - else - echo -e "${RED}โŒ Unhealthy${NC}" - fi -} - -# Function to show logs -show_logs() { - local service=$1 - - if [[ -n "$service" ]]; then - echo -e "${BLUE}๐Ÿ“‹ Logs for $service${NC}" - echo "====================" - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" logs -f "$service" - else - echo -e "${BLUE}๐Ÿ“‹ Logs for all services${NC}" - echo "=======================" - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" logs -f - fi -} - -# Function to run integration tests -run_integration_tests() { - echo -e "${BLUE}๐Ÿงช Running integration tests...${NC}" - - # Make sure services are running - if ! $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" ps | grep -q "Up"; then - echo -e "${RED}โŒ Services are not running. Start them first with '$0 start'${NC}" - exit 1 - fi - - # Run integration tests - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" --profile test run --rm integration-test - - # Show results - if [[ -d "./test-results" ]]; then - echo -e "${BLUE}๐Ÿ“Š Test Results${NC}" - echo "===============" - - if [[ -f "./test-results/overall.result" ]]; then - local result - result=$(cat "./test-results/overall.result") - if [[ "$result" == "SUCCESS" ]]; then - echo -e "${GREEN}๐ŸŽ‰ ALL TESTS PASSED!${NC}" - else - echo -e "${RED}๐Ÿ’ฅ SOME TESTS FAILED!${NC}" - fi - fi - - echo "" - echo "Detailed results available in: ./test-results/" - ls -la ./test-results/ - fi -} - -# Function to run performance tests -run_performance_tests() { - echo -e "${BLUE}๐Ÿ Running performance tests...${NC}" - - # Make sure services are running - if ! $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" ps | grep -q "Up"; then - echo -e "${RED}โŒ Services are not running. Start them first with '$0 start'${NC}" - exit 1 - fi - - # Run performance tests - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" --profile performance run --rm performance-test - - # Show results - if [[ -d "./performance-results" ]]; then - echo -e "${BLUE}๐Ÿ“Š Performance Results${NC}" - echo "======================" - echo "" - echo "Results available in: ./performance-results/" - ls -la ./performance-results/ - - if [[ -f "./performance-results/performance_report.html" ]]; then - echo "" - echo -e "${GREEN}๐Ÿ“„ HTML Report: ./performance-results/performance_report.html${NC}" - fi - fi -} - -# Function to open shell in mount container -open_shell() { - echo -e "${BLUE}๐Ÿš Opening shell in mount container...${NC}" - - if ! $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" ps seaweedfs-mount | grep -q "Up"; then - echo -e "${RED}โŒ Mount container is not running${NC}" - exit 1 - fi - - docker exec -it "${PROJECT_NAME}-seaweedfs-mount-1" /bin/bash -} - -# Function to cleanup everything -cleanup_all() { - echo -e "${BLUE}๐Ÿงน Full cleanup...${NC}" - - # Stop services - $DOCKER_COMPOSE -f "$COMPOSE_FILE" -p "$PROJECT_NAME" down -v --remove-orphans - - # Remove images - echo "Removing Docker images..." - docker images | grep "$PROJECT_NAME" | awk '{print $3}' | xargs -r docker rmi -f - - # Clean up local files - rm -rf bin/ test-results/ performance-results/ - - echo -e "${GREEN}โœ… Full cleanup completed${NC}" -} - -# Main function -main() { - local command=${1:-""} - - # Check Docker Compose availability - check_docker_compose - - case "$command" in - "start") - start_services - ;; - "stop") - stop_services - ;; - "restart") - restart_services - ;; - "status") - show_status - ;; - "logs") - show_logs "${2:-}" - ;; - "test") - run_integration_tests - ;; - "perf") - run_performance_tests - ;; - "shell") - open_shell - ;; - "cleanup") - cleanup_all - ;; - "build") - build_images - ;; - "help"|"-h"|"--help") - show_usage - ;; - "") - show_usage - ;; - *) - echo -e "${RED}โŒ Unknown command: $command${NC}" - echo "" - show_usage - exit 1 - ;; - esac -} - -# Run main function with all arguments -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh b/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh deleted file mode 100755 index 4475365aa..000000000 --- a/seaweedfs-rdma-sidecar/scripts/run-performance-tests.sh +++ /dev/null @@ -1,338 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -# Configuration -MOUNT_POINT=${MOUNT_POINT:-"/mnt/seaweedfs"} -RDMA_SIDECAR_ADDR=${RDMA_SIDECAR_ADDR:-"rdma-sidecar:8081"} -PERFORMANCE_RESULTS_DIR=${PERFORMANCE_RESULTS_DIR:-"/performance-results"} - -# Create results directory -mkdir -p "$PERFORMANCE_RESULTS_DIR" - -# Log file -LOG_FILE="$PERFORMANCE_RESULTS_DIR/performance-test.log" -exec > >(tee -a "$LOG_FILE") -exec 2>&1 - -echo -e "${BLUE}๐Ÿ SEAWEEDFS RDMA MOUNT PERFORMANCE TESTS${NC}" -echo "===========================================" -echo "Mount Point: $MOUNT_POINT" -echo "RDMA Sidecar: $RDMA_SIDECAR_ADDR" -echo "Results Directory: $PERFORMANCE_RESULTS_DIR" -echo "Log File: $LOG_FILE" -echo "" - -# Function to wait for mount to be ready -wait_for_mount() { - local max_attempts=30 - local attempt=1 - - echo -e "${BLUE}โณ Waiting for mount to be ready...${NC}" - - while [[ $attempt -le $max_attempts ]]; do - if mountpoint -q "$MOUNT_POINT" 2>/dev/null && ls "$MOUNT_POINT" >/dev/null 2>&1; then - echo -e "${GREEN}โœ… Mount is ready${NC}" - return 0 - fi - echo " Attempt $attempt/$max_attempts..." - sleep 2 - ((attempt++)) - done - - echo -e "${RED}โŒ Mount failed to be ready${NC}" - return 1 -} - -# Function to get RDMA statistics -get_rdma_stats() { - curl -s "http://$RDMA_SIDECAR_ADDR/stats" 2>/dev/null || echo "{}" -} - -# Function to run dd performance test -run_dd_test() { - local test_name=$1 - local file_size_mb=$2 - local block_size=$3 - local operation=$4 # "write" or "read" - - local test_file="$MOUNT_POINT/perf_test_${test_name}.dat" - local result_file="$PERFORMANCE_RESULTS_DIR/dd_${test_name}.json" - - echo -e "${BLUE}๐Ÿ”ฌ Running DD test: $test_name${NC}" - echo " Size: ${file_size_mb}MB, Block Size: $block_size, Operation: $operation" - - local start_time end_time duration_ms throughput_mbps - - if [[ "$operation" == "write" ]]; then - start_time=$(date +%s%N) - dd if=/dev/zero of="$test_file" bs="$block_size" count=$((file_size_mb * 1024 * 1024 / $(numfmt --from=iec "$block_size"))) 2>/dev/null - end_time=$(date +%s%N) - else - # Create file first if it doesn't exist - if [[ ! -f "$test_file" ]]; then - dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null - fi - start_time=$(date +%s%N) - dd if="$test_file" of=/dev/null bs="$block_size" 2>/dev/null - end_time=$(date +%s%N) - fi - - duration_ms=$(((end_time - start_time) / 1000000)) - throughput_mbps=$(bc <<< "scale=2; $file_size_mb * 1000 / $duration_ms") - - # Save results - cat > "$result_file" << EOF -{ - "test_name": "$test_name", - "operation": "$operation", - "file_size_mb": $file_size_mb, - "block_size": "$block_size", - "duration_ms": $duration_ms, - "throughput_mbps": $throughput_mbps, - "timestamp": "$(date -Iseconds)" -} -EOF - - echo " Duration: ${duration_ms}ms" - echo " Throughput: ${throughput_mbps} MB/s" - echo "" - - # Cleanup write test files - if [[ "$operation" == "write" ]]; then - rm -f "$test_file" - fi -} - -# Function to run FIO performance test -run_fio_test() { - local test_name=$1 - local rw_type=$2 # "read", "write", "randread", "randwrite" - local block_size=$3 - local file_size=$4 - local iodepth=$5 - - local test_file="$MOUNT_POINT/fio_test_${test_name}.dat" - local result_file="$PERFORMANCE_RESULTS_DIR/fio_${test_name}.json" - - echo -e "${BLUE}๐Ÿ”ฌ Running FIO test: $test_name${NC}" - echo " Type: $rw_type, Block Size: $block_size, File Size: $file_size, IO Depth: $iodepth" - - # Run FIO test - fio --name="$test_name" \ - --filename="$test_file" \ - --rw="$rw_type" \ - --bs="$block_size" \ - --size="$file_size" \ - --iodepth="$iodepth" \ - --direct=1 \ - --runtime=30 \ - --time_based \ - --group_reporting \ - --output-format=json \ - --output="$result_file" \ - 2>/dev/null - - # Extract key metrics - if [[ -f "$result_file" ]]; then - local iops throughput_kbps latency_us - iops=$(jq -r '.jobs[0].'"$rw_type"'.iops // 0' "$result_file" 2>/dev/null || echo "0") - throughput_kbps=$(jq -r '.jobs[0].'"$rw_type"'.bw // 0' "$result_file" 2>/dev/null || echo "0") - latency_us=$(jq -r '.jobs[0].'"$rw_type"'.lat_ns.mean // 0' "$result_file" 2>/dev/null || echo "0") - latency_us=$(bc <<< "scale=2; $latency_us / 1000" 2>/dev/null || echo "0") - - echo " IOPS: $iops" - echo " Throughput: $(bc <<< "scale=2; $throughput_kbps / 1024") MB/s" - echo " Average Latency: ${latency_us} ฮผs" - else - echo " FIO test failed or no results" - fi - echo "" - - # Cleanup - rm -f "$test_file" -} - -# Function to run concurrent access test -run_concurrent_test() { - local num_processes=$1 - local file_size_mb=$2 - - echo -e "${BLUE}๐Ÿ”ฌ Running concurrent access test${NC}" - echo " Processes: $num_processes, File Size per Process: ${file_size_mb}MB" - - local start_time end_time duration_ms total_throughput - local pids=() - - start_time=$(date +%s%N) - - # Start concurrent processes - for i in $(seq 1 "$num_processes"); do - ( - local test_file="$MOUNT_POINT/concurrent_test_$i.dat" - dd if=/dev/zero of="$test_file" bs=1M count="$file_size_mb" 2>/dev/null - dd if="$test_file" of=/dev/null bs=1M 2>/dev/null - rm -f "$test_file" - ) & - pids+=($!) - done - - # Wait for all processes to complete - for pid in "${pids[@]}"; do - wait "$pid" - done - - end_time=$(date +%s%N) - duration_ms=$(((end_time - start_time) / 1000000)) - total_throughput=$(bc <<< "scale=2; $num_processes * $file_size_mb * 2 * 1000 / $duration_ms") - - # Save results - cat > "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" << EOF -{ - "test_name": "concurrent_access", - "num_processes": $num_processes, - "file_size_mb_per_process": $file_size_mb, - "total_data_mb": $((num_processes * file_size_mb * 2)), - "duration_ms": $duration_ms, - "total_throughput_mbps": $total_throughput, - "timestamp": "$(date -Iseconds)" -} -EOF - - echo " Duration: ${duration_ms}ms" - echo " Total Throughput: ${total_throughput} MB/s" - echo "" -} - -# Function to generate performance report -generate_report() { - local report_file="$PERFORMANCE_RESULTS_DIR/performance_report.html" - - echo -e "${BLUE}๐Ÿ“Š Generating performance report...${NC}" - - cat > "$report_file" << 'EOF' - - - - SeaweedFS RDMA Mount Performance Report - - - -
-

๐Ÿ SeaweedFS RDMA Mount Performance Report

-

Generated: $(date)

-

Mount Point: $MOUNT_POINT

-

RDMA Sidecar: $RDMA_SIDECAR_ADDR

-
-EOF - - # Add DD test results - echo '

DD Performance Tests

' >> "$report_file" - - for result_file in "$PERFORMANCE_RESULTS_DIR"/dd_*.json; do - if [[ -f "$result_file" ]]; then - local test_name operation file_size_mb block_size throughput_mbps duration_ms - test_name=$(jq -r '.test_name' "$result_file" 2>/dev/null || echo "unknown") - operation=$(jq -r '.operation' "$result_file" 2>/dev/null || echo "unknown") - file_size_mb=$(jq -r '.file_size_mb' "$result_file" 2>/dev/null || echo "0") - block_size=$(jq -r '.block_size' "$result_file" 2>/dev/null || echo "unknown") - throughput_mbps=$(jq -r '.throughput_mbps' "$result_file" 2>/dev/null || echo "0") - duration_ms=$(jq -r '.duration_ms' "$result_file" 2>/dev/null || echo "0") - - echo "" >> "$report_file" - fi - done - - echo '
TestOperationSizeBlock SizeThroughput (MB/s)Duration (ms)
$test_name$operation${file_size_mb}MB$block_size$throughput_mbps$duration_ms
' >> "$report_file" - - # Add FIO test results - echo '

FIO Performance Tests

' >> "$report_file" - echo '

Detailed FIO results are available in individual JSON files.

' >> "$report_file" - - # Add concurrent test results - if [[ -f "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" ]]; then - echo '

Concurrent Access Test

' >> "$report_file" - local num_processes total_throughput duration_ms - num_processes=$(jq -r '.num_processes' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") - total_throughput=$(jq -r '.total_throughput_mbps' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") - duration_ms=$(jq -r '.duration_ms' "$PERFORMANCE_RESULTS_DIR/concurrent_test.json" 2>/dev/null || echo "0") - - echo "

Processes: $num_processes

" >> "$report_file" - echo "

Total Throughput: $total_throughput MB/s

" >> "$report_file" - echo "

Duration: $duration_ms ms

" >> "$report_file" - echo '
' >> "$report_file" - fi - - echo '' >> "$report_file" - - echo " Report saved to: $report_file" -} - -# Main test execution -main() { - echo -e "${BLUE}๐Ÿš€ Starting performance tests...${NC}" - echo "" - - # Wait for mount to be ready - if ! wait_for_mount; then - echo -e "${RED}โŒ Mount is not ready, aborting tests${NC}" - exit 1 - fi - - # Get initial RDMA stats - echo -e "${BLUE}๐Ÿ“Š Initial RDMA Statistics:${NC}" - get_rdma_stats | jq . 2>/dev/null || get_rdma_stats - echo "" - - # Run DD performance tests - echo -e "${BLUE}๐Ÿƒ Running DD Performance Tests...${NC}" - run_dd_test "small_write" 10 "4k" "write" - run_dd_test "small_read" 10 "4k" "read" - run_dd_test "medium_write" 100 "64k" "write" - run_dd_test "medium_read" 100 "64k" "read" - run_dd_test "large_write" 500 "1M" "write" - run_dd_test "large_read" 500 "1M" "read" - - # Run FIO performance tests - echo -e "${BLUE}๐Ÿƒ Running FIO Performance Tests...${NC}" - run_fio_test "seq_read" "read" "64k" "100M" 1 - run_fio_test "seq_write" "write" "64k" "100M" 1 - run_fio_test "rand_read" "randread" "4k" "100M" 16 - run_fio_test "rand_write" "randwrite" "4k" "100M" 16 - - # Run concurrent access test - echo -e "${BLUE}๐Ÿƒ Running Concurrent Access Test...${NC}" - run_concurrent_test 4 50 - - # Get final RDMA stats - echo -e "${BLUE}๐Ÿ“Š Final RDMA Statistics:${NC}" - get_rdma_stats | jq . 2>/dev/null || get_rdma_stats - echo "" - - # Generate performance report - generate_report - - echo -e "${GREEN}๐ŸŽ‰ Performance tests completed!${NC}" - echo "Results saved to: $PERFORMANCE_RESULTS_DIR" -} - -# Run main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/test-complete-optimization.sh b/seaweedfs-rdma-sidecar/scripts/test-complete-optimization.sh deleted file mode 100755 index f9d298461..000000000 --- a/seaweedfs-rdma-sidecar/scripts/test-complete-optimization.sh +++ /dev/null @@ -1,250 +0,0 @@ -#!/bin/bash - -# Complete RDMA Optimization Test -# Demonstrates the full optimization pipeline: Zero-Copy + Connection Pooling + RDMA - -set -e - -echo "๐Ÿ”ฅ SeaweedFS RDMA Complete Optimization Test" -echo "Zero-Copy Page Cache + Connection Pooling + RDMA Bandwidth" -echo "=============================================================" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' - -# Test configuration -SIDECAR_URL="http://localhost:8081" -VOLUME_SERVER="http://seaweedfs-volume:8080" - -# Function to test RDMA sidecar functionality -test_sidecar_health() { - echo -e "\n${CYAN}๐Ÿฅ Testing RDMA Sidecar Health${NC}" - echo "--------------------------------" - - local response=$(curl -s "$SIDECAR_URL/health" 2>/dev/null || echo "{}") - local status=$(echo "$response" | jq -r '.status // "unknown"' 2>/dev/null || echo "unknown") - - if [[ "$status" == "healthy" ]]; then - echo -e "โœ… ${GREEN}Sidecar is healthy${NC}" - - # Check RDMA capabilities - local rdma_enabled=$(echo "$response" | jq -r '.rdma.enabled // false' 2>/dev/null || echo "false") - local zerocopy_enabled=$(echo "$response" | jq -r '.rdma.zerocopy_enabled // false' 2>/dev/null || echo "false") - local pooling_enabled=$(echo "$response" | jq -r '.rdma.pooling_enabled // false' 2>/dev/null || echo "false") - - echo " RDMA enabled: $rdma_enabled" - echo " Zero-copy enabled: $zerocopy_enabled" - echo " Connection pooling enabled: $pooling_enabled" - - return 0 - else - echo -e "โŒ ${RED}Sidecar health check failed${NC}" - return 1 - fi -} - -# Function to test zero-copy optimization -test_zerocopy_optimization() { - echo -e "\n${PURPLE}๐Ÿ”ฅ Testing Zero-Copy Page Cache Optimization${NC}" - echo "----------------------------------------------" - - # Test with a file size above the 64KB threshold - local test_size=1048576 # 1MB - echo "Testing with 1MB file (above 64KB zero-copy threshold)..." - - local response=$(curl -s "$SIDECAR_URL/read?volume=1&needle=1&cookie=1&size=$test_size&volume_server=$VOLUME_SERVER") - - local use_temp_file=$(echo "$response" | jq -r '.use_temp_file // false' 2>/dev/null || echo "false") - local temp_file=$(echo "$response" | jq -r '.temp_file // ""' 2>/dev/null || echo "") - local source=$(echo "$response" | jq -r '.source // "unknown"' 2>/dev/null || echo "unknown") - - if [[ "$use_temp_file" == "true" ]] && [[ -n "$temp_file" ]]; then - echo -e "โœ… ${GREEN}Zero-copy optimization ACTIVE${NC}" - echo " Temp file created: $temp_file" - echo " Source: $source" - return 0 - elif [[ "$source" == *"rdma"* ]]; then - echo -e "โšก ${YELLOW}RDMA active (zero-copy not triggered)${NC}" - echo " Source: $source" - echo " Note: File may be below 64KB threshold or zero-copy disabled" - return 0 - else - echo -e "โŒ ${RED}Zero-copy optimization not detected${NC}" - echo " Response: $response" - return 1 - fi -} - -# Function to test connection pooling -test_connection_pooling() { - echo -e "\n${BLUE}๐Ÿ”Œ Testing RDMA Connection Pooling${NC}" - echo "-----------------------------------" - - echo "Making multiple rapid requests to test connection reuse..." - - local pooled_count=0 - local total_requests=5 - - for i in $(seq 1 $total_requests); do - echo -n " Request $i: " - - local start_time=$(date +%s%N) - local response=$(curl -s "$SIDECAR_URL/read?volume=1&needle=$i&cookie=1&size=65536&volume_server=$VOLUME_SERVER") - local end_time=$(date +%s%N) - - local duration_ns=$((end_time - start_time)) - local duration_ms=$((duration_ns / 1000000)) - - local source=$(echo "$response" | jq -r '.source // "unknown"' 2>/dev/null || echo "unknown") - local session_id=$(echo "$response" | jq -r '.session_id // ""' 2>/dev/null || echo "") - - if [[ "$source" == *"pooled"* ]] || [[ -n "$session_id" ]]; then - pooled_count=$((pooled_count + 1)) - echo -e "${GREEN}${duration_ms}ms (pooled: $session_id)${NC}" - else - echo -e "${YELLOW}${duration_ms}ms (source: $source)${NC}" - fi - - # Small delay to test connection reuse - sleep 0.1 - done - - echo "" - echo "Connection pooling analysis:" - echo " Requests using pooled connections: $pooled_count/$total_requests" - - if [[ $pooled_count -gt 0 ]]; then - echo -e "โœ… ${GREEN}Connection pooling is working${NC}" - return 0 - else - echo -e "โš ๏ธ ${YELLOW}Connection pooling not detected (may be using single connection mode)${NC}" - return 0 - fi -} - -# Function to test performance comparison -test_performance_comparison() { - echo -e "\n${CYAN}โšก Performance Comparison Test${NC}" - echo "-------------------------------" - - local sizes=(65536 262144 1048576) # 64KB, 256KB, 1MB - local size_names=("64KB" "256KB" "1MB") - - for i in "${!sizes[@]}"; do - local size=${sizes[$i]} - local size_name=${size_names[$i]} - - echo "Testing $size_name files:" - - # Test multiple requests to see optimization progression - for j in $(seq 1 3); do - echo -n " Request $j: " - - local start_time=$(date +%s%N) - local response=$(curl -s "$SIDECAR_URL/read?volume=1&needle=$j&cookie=1&size=$size&volume_server=$VOLUME_SERVER") - local end_time=$(date +%s%N) - - local duration_ns=$((end_time - start_time)) - local duration_ms=$((duration_ns / 1000000)) - - local is_rdma=$(echo "$response" | jq -r '.is_rdma // false' 2>/dev/null || echo "false") - local source=$(echo "$response" | jq -r '.source // "unknown"' 2>/dev/null || echo "unknown") - local use_temp_file=$(echo "$response" | jq -r '.use_temp_file // false' 2>/dev/null || echo "false") - - # Color code based on optimization level - if [[ "$source" == "rdma-zerocopy" ]] || [[ "$use_temp_file" == "true" ]]; then - echo -e "${GREEN}${duration_ms}ms (RDMA+ZeroCopy) ๐Ÿ”ฅ${NC}" - elif [[ "$is_rdma" == "true" ]]; then - echo -e "${YELLOW}${duration_ms}ms (RDMA) โšก${NC}" - else - echo -e "โš ๏ธ ${duration_ms}ms (HTTP fallback)" - fi - done - echo "" - done -} - -# Function to test RDMA engine connectivity -test_rdma_engine() { - echo -e "\n${PURPLE}๐Ÿš€ Testing RDMA Engine Connectivity${NC}" - echo "------------------------------------" - - # Get sidecar stats to check RDMA engine connection - local stats_response=$(curl -s "$SIDECAR_URL/stats" 2>/dev/null || echo "{}") - local rdma_connected=$(echo "$stats_response" | jq -r '.rdma.connected // false' 2>/dev/null || echo "false") - - if [[ "$rdma_connected" == "true" ]]; then - echo -e "โœ… ${GREEN}RDMA engine is connected${NC}" - - local total_requests=$(echo "$stats_response" | jq -r '.total_requests // 0' 2>/dev/null || echo "0") - local successful_reads=$(echo "$stats_response" | jq -r '.successful_reads // 0' 2>/dev/null || echo "0") - local total_bytes=$(echo "$stats_response" | jq -r '.total_bytes_read // 0' 2>/dev/null || echo "0") - - echo " Total requests: $total_requests" - echo " Successful reads: $successful_reads" - echo " Total bytes read: $total_bytes" - - return 0 - else - echo -e "โš ๏ธ ${YELLOW}RDMA engine connection status unclear${NC}" - echo " This may be normal if using mock implementation" - return 0 - fi -} - -# Function to display optimization summary -display_optimization_summary() { - echo -e "\n${GREEN}๐ŸŽฏ OPTIMIZATION SUMMARY${NC}" - echo "========================================" - echo "" - echo -e "${PURPLE}Implemented Optimizations:${NC}" - echo "1. ๐Ÿ”ฅ Zero-Copy Page Cache" - echo " - Eliminates 4 out of 5 memory copies" - echo " - Direct page cache population via temp files" - echo " - Threshold: 64KB+ files" - echo "" - echo "2. ๐Ÿ”Œ RDMA Connection Pooling" - echo " - Eliminates 100ms connection setup cost" - echo " - Reuses connections across requests" - echo " - Automatic cleanup of idle connections" - echo "" - echo "3. โšก RDMA Bandwidth Advantage" - echo " - High-throughput data transfer" - echo " - Bypasses kernel network stack" - echo " - Direct memory access" - echo "" - echo -e "${CYAN}Expected Performance Gains:${NC}" - echo "โ€ข Small files (< 64KB): ~50x improvement from RDMA + pooling" - echo "โ€ข Medium files (64KB-1MB): ~47x improvement from zero-copy + pooling" - echo "โ€ข Large files (> 1MB): ~118x improvement from all optimizations" - echo "" - echo -e "${GREEN}๐Ÿš€ This represents a fundamental breakthrough in distributed storage performance!${NC}" -} - -# Main execution -main() { - echo -e "\n${YELLOW}๐Ÿ”ง Starting comprehensive optimization test...${NC}" - - # Run all tests - test_sidecar_health || exit 1 - test_rdma_engine - test_zerocopy_optimization - test_connection_pooling - test_performance_comparison - display_optimization_summary - - echo -e "\n${GREEN}๐ŸŽ‰ Complete optimization test finished!${NC}" - echo "" - echo "Next steps:" - echo "1. Run performance benchmark: ./scripts/performance-benchmark.sh" - echo "2. Test with weed mount: docker compose -f docker-compose.mount-rdma.yml logs seaweedfs-mount" - echo "3. Monitor connection pool: curl -s http://localhost:8081/stats | jq" -} - -# Execute main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/test-complete-optimizations.sh b/seaweedfs-rdma-sidecar/scripts/test-complete-optimizations.sh deleted file mode 100755 index b84d429fa..000000000 --- a/seaweedfs-rdma-sidecar/scripts/test-complete-optimizations.sh +++ /dev/null @@ -1,295 +0,0 @@ -#!/bin/bash - -# Complete RDMA Optimization Test Suite -# Tests all three optimizations: Zero-Copy + Connection Pooling + RDMA - -set -e - -echo "๐Ÿš€ Complete RDMA Optimization Test Suite" -echo "========================================" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -RED='\033[0;31m' -NC='\033[0m' - -# Test results tracking -TESTS_PASSED=0 -TESTS_TOTAL=0 - -# Helper function to run a test -run_test() { - local test_name="$1" - local test_command="$2" - - ((TESTS_TOTAL++)) - echo -e "\n${CYAN}๐Ÿงช Test $TESTS_TOTAL: $test_name${NC}" - echo "$(printf '%.0s-' {1..50})" - - if eval "$test_command"; then - echo -e "${GREEN}โœ… PASSED: $test_name${NC}" - ((TESTS_PASSED++)) - return 0 - else - echo -e "${RED}โŒ FAILED: $test_name${NC}" - return 1 - fi -} - -# Test 1: Build verification -test_build_verification() { - echo "๐Ÿ“ฆ Verifying all components build successfully..." - - # Check demo server binary - if [[ -f "bin/demo-server" ]]; then - echo "โœ… Demo server binary exists" - else - echo "โŒ Demo server binary missing" - return 1 - fi - - # Check RDMA engine binary - if [[ -f "rdma-engine/target/release/rdma-engine-server" ]]; then - echo "โœ… RDMA engine binary exists" - else - echo "โŒ RDMA engine binary missing" - return 1 - fi - - # Check SeaweedFS binary - if [[ -f "../weed/weed" ]]; then - echo "โœ… SeaweedFS with RDMA support exists" - else - echo "โŒ SeaweedFS binary missing (expected at ../weed/weed)" - return 1 - fi - - echo "๐ŸŽฏ All core components built successfully" - return 0 -} - -# Test 2: Zero-copy mechanism -test_zero_copy_mechanism() { - echo "๐Ÿ”ฅ Testing zero-copy page cache mechanism..." - - local temp_dir="/tmp/rdma-test-$$" - mkdir -p "$temp_dir" - - # Create test data - local test_file="$temp_dir/test_data.bin" - dd if=/dev/urandom of="$test_file" bs=1024 count=64 2>/dev/null - - # Simulate temp file creation (sidecar behavior) - local temp_needle="$temp_dir/vol1_needle123.tmp" - cp "$test_file" "$temp_needle" - - if [[ -f "$temp_needle" ]]; then - echo "โœ… Temp file created successfully" - - # Simulate reading (mount behavior) - local read_result="$temp_dir/read_result.bin" - cp "$temp_needle" "$read_result" - - if cmp -s "$test_file" "$read_result"; then - echo "โœ… Zero-copy read successful with data integrity" - rm -rf "$temp_dir" - return 0 - else - echo "โŒ Data integrity check failed" - rm -rf "$temp_dir" - return 1 - fi - else - echo "โŒ Temp file creation failed" - rm -rf "$temp_dir" - return 1 - fi -} - -# Test 3: Connection pooling logic -test_connection_pooling() { - echo "๐Ÿ”Œ Testing connection pooling logic..." - - # Test the core pooling mechanism by running our pool test - local pool_test_output - pool_test_output=$(./scripts/test-connection-pooling.sh 2>&1 | tail -20) - - if echo "$pool_test_output" | grep -q "Connection pool test completed successfully"; then - echo "โœ… Connection pooling logic verified" - return 0 - else - echo "โŒ Connection pooling test failed" - return 1 - fi -} - -# Test 4: Configuration validation -test_configuration_validation() { - echo "โš™๏ธ Testing configuration validation..." - - # Test demo server help - if ./bin/demo-server --help | grep -q "enable-zerocopy"; then - echo "โœ… Zero-copy configuration available" - else - echo "โŒ Zero-copy configuration missing" - return 1 - fi - - if ./bin/demo-server --help | grep -q "enable-pooling"; then - echo "โœ… Connection pooling configuration available" - else - echo "โŒ Connection pooling configuration missing" - return 1 - fi - - if ./bin/demo-server --help | grep -q "max-connections"; then - echo "โœ… Pool sizing configuration available" - else - echo "โŒ Pool sizing configuration missing" - return 1 - fi - - echo "๐ŸŽฏ All configuration options validated" - return 0 -} - -# Test 5: RDMA engine mock functionality -test_rdma_engine_mock() { - echo "๐Ÿš€ Testing RDMA engine mock functionality..." - - # Start RDMA engine in background for quick test - local engine_log="/tmp/rdma-engine-test.log" - local socket_path="/tmp/rdma-test-engine.sock" - - # Clean up any existing socket - rm -f "$socket_path" - - # Start engine in background - timeout 10s ./rdma-engine/target/release/rdma-engine-server \ - --ipc-socket "$socket_path" \ - --debug > "$engine_log" 2>&1 & - - local engine_pid=$! - - # Wait a moment for startup - sleep 2 - - # Check if socket was created - if [[ -S "$socket_path" ]]; then - echo "โœ… RDMA engine socket created successfully" - kill $engine_pid 2>/dev/null || true - wait $engine_pid 2>/dev/null || true - rm -f "$socket_path" "$engine_log" - return 0 - else - echo "โŒ RDMA engine socket not created" - kill $engine_pid 2>/dev/null || true - wait $engine_pid 2>/dev/null || true - echo "Engine log:" - cat "$engine_log" 2>/dev/null || echo "No log available" - rm -f "$socket_path" "$engine_log" - return 1 - fi -} - -# Test 6: Integration test preparation -test_integration_readiness() { - echo "๐Ÿงฉ Testing integration readiness..." - - # Check Docker Compose file - if [[ -f "docker-compose.mount-rdma.yml" ]]; then - echo "โœ… Docker Compose configuration available" - else - echo "โŒ Docker Compose configuration missing" - return 1 - fi - - # Validate Docker Compose syntax - if docker compose -f docker-compose.mount-rdma.yml config > /dev/null 2>&1; then - echo "โœ… Docker Compose configuration valid" - else - echo "โŒ Docker Compose configuration invalid" - return 1 - fi - - # Check test scripts - local scripts=("test-zero-copy-mechanism.sh" "test-connection-pooling.sh" "performance-benchmark.sh") - for script in "${scripts[@]}"; do - if [[ -x "scripts/$script" ]]; then - echo "โœ… Test script available: $script" - else - echo "โŒ Test script missing or not executable: $script" - return 1 - fi - done - - echo "๐ŸŽฏ Integration environment ready" - return 0 -} - -# Performance benchmarking -test_performance_characteristics() { - echo "๐Ÿ“Š Testing performance characteristics..." - - # Run zero-copy performance test - if ./scripts/test-zero-copy-mechanism.sh | grep -q "Performance improvement"; then - echo "โœ… Zero-copy performance improvement detected" - else - echo "โŒ Zero-copy performance test failed" - return 1 - fi - - echo "๐ŸŽฏ Performance characteristics validated" - return 0 -} - -# Main test execution -main() { - echo -e "${BLUE}๐Ÿš€ Starting complete optimization test suite...${NC}" - echo "" - - # Run all tests - run_test "Build Verification" "test_build_verification" - run_test "Zero-Copy Mechanism" "test_zero_copy_mechanism" - run_test "Connection Pooling" "test_connection_pooling" - run_test "Configuration Validation" "test_configuration_validation" - run_test "RDMA Engine Mock" "test_rdma_engine_mock" - run_test "Integration Readiness" "test_integration_readiness" - run_test "Performance Characteristics" "test_performance_characteristics" - - # Results summary - echo -e "\n${PURPLE}๐Ÿ“Š Test Results Summary${NC}" - echo "=======================" - echo "Tests passed: $TESTS_PASSED/$TESTS_TOTAL" - - if [[ $TESTS_PASSED -eq $TESTS_TOTAL ]]; then - echo -e "${GREEN}๐ŸŽ‰ ALL TESTS PASSED!${NC}" - echo "" - echo -e "${CYAN}๐Ÿš€ Revolutionary Optimization Suite Status:${NC}" - echo "โœ… Zero-Copy Page Cache: WORKING" - echo "โœ… RDMA Connection Pooling: WORKING" - echo "โœ… RDMA Engine Integration: WORKING" - echo "โœ… Mount Client Integration: READY" - echo "โœ… Docker Environment: READY" - echo "โœ… Performance Testing: READY" - echo "" - echo -e "${YELLOW}๐Ÿ”ฅ Expected Performance Improvements:${NC}" - echo "โ€ข Small files (< 64KB): 50x faster" - echo "โ€ข Medium files (64KB-1MB): 47x faster" - echo "โ€ข Large files (> 1MB): 118x faster" - echo "" - echo -e "${GREEN}Ready for production testing! ๐Ÿš€${NC}" - return 0 - else - echo -e "${RED}โŒ SOME TESTS FAILED${NC}" - echo "Please review the failed tests above" - return 1 - fi -} - -# Execute main function -main "$@" diff --git a/seaweedfs-rdma-sidecar/scripts/test-connection-pooling.sh b/seaweedfs-rdma-sidecar/scripts/test-connection-pooling.sh deleted file mode 100755 index 576b905c0..000000000 --- a/seaweedfs-rdma-sidecar/scripts/test-connection-pooling.sh +++ /dev/null @@ -1,209 +0,0 @@ -#!/bin/bash - -# Test RDMA Connection Pooling Mechanism -# Demonstrates connection reuse and pool management - -set -e - -echo "๐Ÿ”Œ Testing RDMA Connection Pooling Mechanism" -echo "============================================" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -NC='\033[0m' - -echo -e "\n${BLUE}๐Ÿงช Testing Connection Pool Logic${NC}" -echo "--------------------------------" - -# Test the pool implementation by building a simple test -cat > /tmp/pool_test.go << 'EOF' -package main - -import ( - "context" - "fmt" - "time" -) - -// Simulate the connection pool behavior -type PooledConnection struct { - ID string - lastUsed time.Time - inUse bool - created time.Time -} - -type ConnectionPool struct { - connections []*PooledConnection - maxConnections int - maxIdleTime time.Duration -} - -func NewConnectionPool(maxConnections int, maxIdleTime time.Duration) *ConnectionPool { - return &ConnectionPool{ - connections: make([]*PooledConnection, 0, maxConnections), - maxConnections: maxConnections, - maxIdleTime: maxIdleTime, - } -} - -func (p *ConnectionPool) getConnection() (*PooledConnection, error) { - // Look for available connection - for _, conn := range p.connections { - if !conn.inUse && time.Since(conn.lastUsed) < p.maxIdleTime { - conn.inUse = true - conn.lastUsed = time.Now() - fmt.Printf("๐Ÿ”„ Reusing connection: %s (age: %v)\n", conn.ID, time.Since(conn.created)) - return conn, nil - } - } - - // Create new connection if under limit - if len(p.connections) < p.maxConnections { - conn := &PooledConnection{ - ID: fmt.Sprintf("conn-%d-%d", len(p.connections), time.Now().Unix()), - lastUsed: time.Now(), - inUse: true, - created: time.Now(), - } - p.connections = append(p.connections, conn) - fmt.Printf("๐Ÿš€ Created new connection: %s (pool size: %d)\n", conn.ID, len(p.connections)) - return conn, nil - } - - return nil, fmt.Errorf("pool exhausted (max: %d)", p.maxConnections) -} - -func (p *ConnectionPool) releaseConnection(conn *PooledConnection) { - conn.inUse = false - conn.lastUsed = time.Now() - fmt.Printf("๐Ÿ”“ Released connection: %s\n", conn.ID) -} - -func (p *ConnectionPool) cleanup() { - now := time.Now() - activeConnections := make([]*PooledConnection, 0, len(p.connections)) - - for _, conn := range p.connections { - if conn.inUse || now.Sub(conn.lastUsed) < p.maxIdleTime { - activeConnections = append(activeConnections, conn) - } else { - fmt.Printf("๐Ÿงน Cleaned up idle connection: %s (idle: %v)\n", conn.ID, now.Sub(conn.lastUsed)) - } - } - - p.connections = activeConnections -} - -func (p *ConnectionPool) getStats() (int, int) { - total := len(p.connections) - inUse := 0 - for _, conn := range p.connections { - if conn.inUse { - inUse++ - } - } - return total, inUse -} - -func main() { - fmt.Println("๐Ÿ”Œ Connection Pool Test Starting...") - - // Create pool with small limits for testing - pool := NewConnectionPool(3, 2*time.Second) - - fmt.Println("\n1. Testing connection creation and reuse:") - - // Get multiple connections - conns := make([]*PooledConnection, 0) - for i := 0; i < 5; i++ { - conn, err := pool.getConnection() - if err != nil { - fmt.Printf("โŒ Error getting connection %d: %v\n", i+1, err) - continue - } - conns = append(conns, conn) - - // Simulate work - time.Sleep(100 * time.Millisecond) - } - - total, inUse := pool.getStats() - fmt.Printf("\n๐Ÿ“Š Pool stats: %d total connections, %d in use\n", total, inUse) - - fmt.Println("\n2. Testing connection release and reuse:") - - // Release some connections - for i := 0; i < 2; i++ { - if i < len(conns) { - pool.releaseConnection(conns[i]) - } - } - - // Try to get new connections (should reuse) - for i := 0; i < 2; i++ { - conn, err := pool.getConnection() - if err != nil { - fmt.Printf("โŒ Error getting reused connection: %v\n", err) - } else { - pool.releaseConnection(conn) - } - } - - fmt.Println("\n3. Testing cleanup of idle connections:") - - // Wait for connections to become idle - fmt.Println("โฑ๏ธ Waiting for connections to become idle...") - time.Sleep(3 * time.Second) - - // Cleanup - pool.cleanup() - - total, inUse = pool.getStats() - fmt.Printf("๐Ÿ“Š Pool stats after cleanup: %d total connections, %d in use\n", total, inUse) - - fmt.Println("\nโœ… Connection pool test completed successfully!") - fmt.Println("\n๐ŸŽฏ Key benefits demonstrated:") - fmt.Println(" โ€ข Connection reuse eliminates setup cost") - fmt.Println(" โ€ข Pool size limits prevent resource exhaustion") - fmt.Println(" โ€ข Automatic cleanup prevents memory leaks") - fmt.Println(" โ€ข Idle timeout ensures fresh connections") -} -EOF - -echo "๐Ÿ“ Created connection pool test program" - -echo -e "\n${GREEN}๐Ÿš€ Running connection pool simulation${NC}" -echo "------------------------------------" - -# Run the test -cd /tmp && go run pool_test.go - -echo -e "\n${YELLOW}๐Ÿ“Š Performance Impact Analysis${NC}" -echo "------------------------------" - -echo "Without connection pooling:" -echo " โ€ข Each request: 100ms setup + 1ms transfer = 101ms" -echo " โ€ข 10 requests: 10 ร— 101ms = 1010ms" - -echo "" -echo "With connection pooling:" -echo " โ€ข First request: 100ms setup + 1ms transfer = 101ms" -echo " โ€ข Next 9 requests: 0.1ms reuse + 1ms transfer = 1.1ms each" -echo " โ€ข 10 requests: 101ms + (9 ร— 1.1ms) = 111ms" - -echo "" -echo -e "${GREEN}๐Ÿ”ฅ Performance improvement: 1010ms โ†’ 111ms = 9x faster!${NC}" - -echo -e "\n${PURPLE}๐Ÿ’ก Real-world scaling benefits:${NC}" -echo "โ€ข 100 requests: 100x faster with pooling" -echo "โ€ข 1000 requests: 1000x faster with pooling" -echo "โ€ข Connection pool amortizes setup cost across many operations" - -# Cleanup -rm -f /tmp/pool_test.go - -echo -e "\n${GREEN}โœ… Connection pooling test completed!${NC}" diff --git a/seaweedfs-rdma-sidecar/scripts/test-zero-copy-mechanism.sh b/seaweedfs-rdma-sidecar/scripts/test-zero-copy-mechanism.sh deleted file mode 100755 index 63c5d3584..000000000 --- a/seaweedfs-rdma-sidecar/scripts/test-zero-copy-mechanism.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash - -# Test Zero-Copy Page Cache Mechanism -# Demonstrates the core innovation without needing full server - -set -e - -echo "๐Ÿ”ฅ Testing Zero-Copy Page Cache Mechanism" -echo "=========================================" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -NC='\033[0m' - -# Test configuration -TEMP_DIR="/tmp/rdma-cache-test" -TEST_DATA_SIZE=1048576 # 1MB -ITERATIONS=5 - -# Cleanup function -cleanup() { - rm -rf "$TEMP_DIR" 2>/dev/null || true -} - -# Setup -setup() { - echo -e "\n${BLUE}๐Ÿ”ง Setting up test environment${NC}" - cleanup - mkdir -p "$TEMP_DIR" - echo "โœ… Created temp directory: $TEMP_DIR" -} - -# Generate test data -generate_test_data() { - echo -e "\n${PURPLE}๐Ÿ“ Generating test data${NC}" - dd if=/dev/urandom of="$TEMP_DIR/source_data.bin" bs=$TEST_DATA_SIZE count=1 2>/dev/null - echo "โœ… Generated $TEST_DATA_SIZE bytes of test data" -} - -# Test 1: Simulate the zero-copy write mechanism -test_zero_copy_write() { - echo -e "\n${GREEN}๐Ÿ”ฅ Test 1: Zero-Copy Page Cache Population${NC}" - echo "--------------------------------------------" - - local source_file="$TEMP_DIR/source_data.bin" - local temp_file="$TEMP_DIR/vol1_needle123_cookie456.tmp" - - echo "๐Ÿ“ค Simulating RDMA sidecar writing to temp file..." - - # This simulates what our sidecar does: - # ioutil.WriteFile(tempFilePath, data, 0644) - local start_time=$(date +%s%N) - cp "$source_file" "$temp_file" - local end_time=$(date +%s%N) - - local write_duration_ns=$((end_time - start_time)) - local write_duration_ms=$((write_duration_ns / 1000000)) - - echo "โœ… Temp file written in ${write_duration_ms}ms" - echo " File: $temp_file" - echo " Size: $(stat -f%z "$temp_file" 2>/dev/null || stat -c%s "$temp_file") bytes" - - # Check if file is in page cache (approximation) - if command -v vmtouch >/dev/null 2>&1; then - echo " Page cache status:" - vmtouch "$temp_file" 2>/dev/null || echo " (vmtouch not available for precise measurement)" - else - echo " ๐Ÿ“„ File written to filesystem (page cache populated automatically)" - fi -} - -# Test 2: Simulate the zero-copy read mechanism -test_zero_copy_read() { - echo -e "\n${GREEN}โšก Test 2: Zero-Copy Page Cache Read${NC}" - echo "-----------------------------------" - - local temp_file="$TEMP_DIR/vol1_needle123_cookie456.tmp" - local read_buffer="$TEMP_DIR/read_buffer.bin" - - echo "๐Ÿ“ฅ Simulating mount client reading from temp file..." - - # This simulates what our mount client does: - # file.Read(buffer) from temp file - local start_time=$(date +%s%N) - - # Multiple reads to test page cache efficiency - for i in $(seq 1 $ITERATIONS); do - cp "$temp_file" "$read_buffer.tmp$i" - done - - local end_time=$(date +%s%N) - local read_duration_ns=$((end_time - start_time)) - local read_duration_ms=$((read_duration_ns / 1000000)) - local avg_read_ms=$((read_duration_ms / ITERATIONS)) - - echo "โœ… $ITERATIONS reads completed in ${read_duration_ms}ms" - echo " Average per read: ${avg_read_ms}ms" - echo " ๐Ÿ”ฅ Subsequent reads served from page cache!" - - # Verify data integrity - if cmp -s "$TEMP_DIR/source_data.bin" "$read_buffer.tmp1"; then - echo "โœ… Data integrity verified - zero corruption" - else - echo "โŒ Data integrity check failed" - return 1 - fi -} - -# Test 3: Performance comparison -test_performance_comparison() { - echo -e "\n${YELLOW}๐Ÿ“Š Test 3: Performance Comparison${NC}" - echo "-----------------------------------" - - local source_file="$TEMP_DIR/source_data.bin" - - echo "๐ŸŒ Traditional copy (simulating multiple memory copies):" - local start_time=$(date +%s%N) - - # Simulate 5 memory copies (traditional path) - cp "$source_file" "$TEMP_DIR/copy1.bin" - cp "$TEMP_DIR/copy1.bin" "$TEMP_DIR/copy2.bin" - cp "$TEMP_DIR/copy2.bin" "$TEMP_DIR/copy3.bin" - cp "$TEMP_DIR/copy3.bin" "$TEMP_DIR/copy4.bin" - cp "$TEMP_DIR/copy4.bin" "$TEMP_DIR/copy5.bin" - - local end_time=$(date +%s%N) - local traditional_duration_ns=$((end_time - start_time)) - local traditional_duration_ms=$((traditional_duration_ns / 1000000)) - - echo " 5 memory copies: ${traditional_duration_ms}ms" - - echo "๐Ÿš€ Zero-copy method (page cache):" - local start_time=$(date +%s%N) - - # Simulate zero-copy path (write once, read multiple times from cache) - cp "$source_file" "$TEMP_DIR/zerocopy.tmp" - # Subsequent reads are from page cache - cp "$TEMP_DIR/zerocopy.tmp" "$TEMP_DIR/result.bin" - - local end_time=$(date +%s%N) - local zerocopy_duration_ns=$((end_time - start_time)) - local zerocopy_duration_ms=$((zerocopy_duration_ns / 1000000)) - - echo " Write + cached read: ${zerocopy_duration_ms}ms" - - # Calculate improvement - if [[ $zerocopy_duration_ms -gt 0 ]]; then - local improvement=$((traditional_duration_ms / zerocopy_duration_ms)) - echo "" - echo -e "${GREEN}๐ŸŽฏ Performance improvement: ${improvement}x faster${NC}" - - if [[ $improvement -gt 5 ]]; then - echo -e "${GREEN}๐Ÿ”ฅ EXCELLENT: Significant optimization detected!${NC}" - elif [[ $improvement -gt 2 ]]; then - echo -e "${YELLOW}โšก GOOD: Measurable improvement${NC}" - else - echo -e "${YELLOW}๐Ÿ“ˆ MODERATE: Some improvement (limited by I/O overhead)${NC}" - fi - fi -} - -# Test 4: Demonstrate temp file cleanup with persistent page cache -test_cleanup_behavior() { - echo -e "\n${PURPLE}๐Ÿงน Test 4: Cleanup with Page Cache Persistence${NC}" - echo "----------------------------------------------" - - local temp_file="$TEMP_DIR/cleanup_test.tmp" - - # Write data - echo "๐Ÿ“ Writing data to temp file..." - cp "$TEMP_DIR/source_data.bin" "$temp_file" - - # Read to ensure it's in page cache - echo "๐Ÿ“– Reading data (loads into page cache)..." - cp "$temp_file" "$TEMP_DIR/cache_load.bin" - - # Delete temp file (simulating our cleanup) - echo "๐Ÿ—‘๏ธ Deleting temp file (simulating cleanup)..." - rm "$temp_file" - - # Try to access page cache data (this would work in real scenario) - echo "๐Ÿ” File deleted but page cache may still contain data" - echo " (In real implementation, this provides brief performance window)" - - if [[ -f "$TEMP_DIR/cache_load.bin" ]]; then - echo "โœ… Data successfully accessed from loaded cache" - fi - - echo "" - echo -e "${BLUE}๐Ÿ’ก Key insight: Page cache persists briefly even after file deletion${NC}" - echo " This allows zero-copy reads during the critical performance window" -} - -# Main execution -main() { - echo -e "${BLUE}๐Ÿš€ Starting zero-copy mechanism test...${NC}" - - setup - generate_test_data - test_zero_copy_write - test_zero_copy_read - test_performance_comparison - test_cleanup_behavior - - echo -e "\n${GREEN}๐ŸŽ‰ Zero-copy mechanism test completed!${NC}" - echo "" - echo -e "${PURPLE}๐Ÿ“‹ Summary of what we demonstrated:${NC}" - echo "1. โœ… Temp file write populates page cache automatically" - echo "2. โœ… Subsequent reads served from fast page cache" - echo "3. โœ… Significant performance improvement over multiple copies" - echo "4. โœ… Cleanup behavior maintains performance window" - echo "" - echo -e "${YELLOW}๐Ÿ”ฅ This is the core mechanism behind our 100x performance improvement!${NC}" - - cleanup -} - -# Run the test -main "$@" diff --git a/seaweedfs-rdma-sidecar/sidecar b/seaweedfs-rdma-sidecar/sidecar deleted file mode 100755 index daddfdbf1..000000000 Binary files a/seaweedfs-rdma-sidecar/sidecar and /dev/null differ diff --git a/seaweedfs-rdma-sidecar/test-fixes-standalone.go b/seaweedfs-rdma-sidecar/test-fixes-standalone.go deleted file mode 100644 index 5b709bc7b..000000000 --- a/seaweedfs-rdma-sidecar/test-fixes-standalone.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "fmt" - "strconv" - "strings" -) - -// Test the improved parse functions (from cmd/sidecar/main.go fix) -func parseUint32(s string, defaultValue uint32) uint32 { - if s == "" { - return defaultValue - } - val, err := strconv.ParseUint(s, 10, 32) - if err != nil { - return defaultValue - } - return uint32(val) -} - -func parseUint64(s string, defaultValue uint64) uint64 { - if s == "" { - return defaultValue - } - val, err := strconv.ParseUint(s, 10, 64) - if err != nil { - return defaultValue - } - return val -} - -// Test the improved error reporting pattern (from weed/mount/rdma_client.go fix) -func testErrorReporting() { - fmt.Println("Testing Error Reporting Fix:") - - // Simulate RDMA failure followed by HTTP failure - rdmaErr := fmt.Errorf("RDMA connection timeout") - httpErr := fmt.Errorf("HTTP 404 Not Found") - - // OLD (incorrect) way: - oldError := fmt.Errorf("both RDMA and HTTP fallback failed: RDMA=%v, HTTP=%v", rdmaErr, rdmaErr) // BUG: same error twice - fmt.Printf(" Old (buggy): %v\n", oldError) - - // NEW (fixed) way: - newError := fmt.Errorf("both RDMA and HTTP fallback failed: RDMA=%v, HTTP=%v", rdmaErr, httpErr) // FIXED: different errors - fmt.Printf(" New (fixed): %v\n", newError) -} - -// Test weed mount command with RDMA flags (from docker-compose fix) -func testWeedMountCommand() { - fmt.Println("Testing Weed Mount Command Fix:") - - // OLD (missing RDMA flags): - oldCommand := "/usr/local/bin/weed mount -filer=seaweedfs-filer:8888 -dir=/mnt/seaweedfs -allowOthers=true -debug" - fmt.Printf(" Old (missing RDMA): %s\n", oldCommand) - - // NEW (with RDMA flags): - newCommand := "/usr/local/bin/weed mount -filer=${FILER_ADDR} -dir=${MOUNT_POINT} -allowOthers=true -rdma.enabled=${RDMA_ENABLED} -rdma.sidecar=${RDMA_SIDECAR_ADDR} -rdma.fallback=${RDMA_FALLBACK} -rdma.maxConcurrent=${RDMA_MAX_CONCURRENT} -rdma.timeoutMs=${RDMA_TIMEOUT_MS} -debug=${DEBUG}" - fmt.Printf(" New (with RDMA): %s\n", newCommand) - - // Check if RDMA flags are present - rdmaFlags := []string{"-rdma.enabled", "-rdma.sidecar", "-rdma.fallback", "-rdma.maxConcurrent", "-rdma.timeoutMs"} - allPresent := true - for _, flag := range rdmaFlags { - if !strings.Contains(newCommand, flag) { - allPresent = false - break - } - } - - if allPresent { - fmt.Println(" All RDMA flags present in command") - } else { - fmt.Println(" Missing RDMA flags") - } -} - -// Test health check robustness (from Dockerfile.rdma-engine fix) -func testHealthCheck() { - fmt.Println("Testing Health Check Fix:") - - // OLD (hardcoded): - oldHealthCheck := "test -S /tmp/rdma-engine.sock" - fmt.Printf(" Old (hardcoded): %s\n", oldHealthCheck) - - // NEW (robust): - newHealthCheck := `pgrep rdma-engine-server >/dev/null && test -d /tmp/rdma && test "$(find /tmp/rdma -name '*.sock' | wc -l)" -gt 0` - fmt.Printf(" New (robust): %s\n", newHealthCheck) -} - -func main() { - fmt.Println("Testing All GitHub PR Review Fixes") - fmt.Println("====================================") - fmt.Println() - - // Test parse functions - fmt.Println("Testing Parse Functions Fix:") - fmt.Printf(" parseUint32('123', 0) = %d (expected: 123)\n", parseUint32("123", 0)) - fmt.Printf(" parseUint32('', 999) = %d (expected: 999)\n", parseUint32("", 999)) - fmt.Printf(" parseUint32('invalid', 999) = %d (expected: 999)\n", parseUint32("invalid", 999)) - fmt.Printf(" parseUint64('12345678901234', 0) = %d (expected: 12345678901234)\n", parseUint64("12345678901234", 0)) - fmt.Printf(" parseUint64('invalid', 999) = %d (expected: 999)\n", parseUint64("invalid", 999)) - fmt.Println(" Parse functions handle errors correctly!") - fmt.Println() - - testErrorReporting() - fmt.Println() - - testWeedMountCommand() - fmt.Println() - - testHealthCheck() - fmt.Println() - - fmt.Println("All Review Fixes Validated!") - fmt.Println("=============================") - fmt.Println() - fmt.Println("Parse functions: Safe error handling with strconv.ParseUint") - fmt.Println("Error reporting: Proper distinction between RDMA and HTTP errors") - fmt.Println("Weed mount: RDMA flags properly included in Docker command") - fmt.Println("Health check: Robust socket detection without hardcoding") - fmt.Println("File ID parsing: Reuses existing SeaweedFS functions") - fmt.Println("Semaphore handling: No more channel close panics") - fmt.Println("Go.mod documentation: Clear instructions for contributors") - fmt.Println() - fmt.Println("Ready for production deployment!") -} diff --git a/seaweedfs-rdma-sidecar/test-rdma-integration.sh b/seaweedfs-rdma-sidecar/test-rdma-integration.sh deleted file mode 100644 index 4b599d3a1..000000000 --- a/seaweedfs-rdma-sidecar/test-rdma-integration.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/bin/bash -set -e - -echo "๐Ÿš€ Testing RDMA Integration with All Fixes Applied" -echo "==================================================" - -# Build the sidecar with all fixes -echo "๐Ÿ“ฆ Building RDMA sidecar..." -go build -o bin/demo-server ./cmd/demo-server -go build -o bin/sidecar ./cmd/sidecar - -# Test that the parse functions work correctly -echo "๐Ÿงช Testing parse helper functions..." -cat > test_parse_functions.go << 'EOF' -package main - -import ( - "fmt" - "strconv" -) - -func parseUint32(s string, defaultValue uint32) uint32 { - if s == "" { - return defaultValue - } - val, err := strconv.ParseUint(s, 10, 32) - if err != nil { - return defaultValue - } - return uint32(val) -} - -func parseUint64(s string, defaultValue uint64) uint64 { - if s == "" { - return defaultValue - } - val, err := strconv.ParseUint(s, 10, 64) - if err != nil { - return defaultValue - } - return val -} - -func main() { - fmt.Println("Testing parseUint32:") - fmt.Printf(" '123' -> %d (expected: 123)\n", parseUint32("123", 0)) - fmt.Printf(" '' -> %d (expected: 999)\n", parseUint32("", 999)) - fmt.Printf(" 'invalid' -> %d (expected: 999)\n", parseUint32("invalid", 999)) - - fmt.Println("Testing parseUint64:") - fmt.Printf(" '12345678901234' -> %d (expected: 12345678901234)\n", parseUint64("12345678901234", 0)) - fmt.Printf(" '' -> %d (expected: 999)\n", parseUint64("", 999)) - fmt.Printf(" 'invalid' -> %d (expected: 999)\n", parseUint64("invalid", 999)) -} -EOF - -go run test_parse_functions.go -rm test_parse_functions.go - -echo "โœ… Parse functions working correctly!" - -# Test the sidecar startup -echo "๐Ÿ Testing sidecar startup..." -timeout 5 ./bin/demo-server --port 8081 --enable-rdma=false --debug --volume-server=http://httpbin.org/get & -SIDECAR_PID=$! - -sleep 2 - -# Test health endpoint -echo "๐Ÿฅ Testing health endpoint..." -if curl -s http://localhost:8081/health | grep -q "healthy"; then - echo "โœ… Health endpoint working!" -else - echo "โŒ Health endpoint failed!" -fi - -# Test stats endpoint -echo "๐Ÿ“Š Testing stats endpoint..." -if curl -s http://localhost:8081/stats | jq . > /dev/null; then - echo "โœ… Stats endpoint working!" -else - echo "โŒ Stats endpoint failed!" -fi - -# Test read endpoint (will fallback to HTTP) -echo "๐Ÿ“– Testing read endpoint..." -RESPONSE=$(curl -s "http://localhost:8081/read?volume=1&needle=123&cookie=456&offset=0&size=1024&volume_server=http://localhost:8080") -if echo "$RESPONSE" | jq . > /dev/null; then - echo "โœ… Read endpoint working!" - echo " Response structure valid JSON" - - # Check if it has the expected fields - if echo "$RESPONSE" | jq -e '.source' > /dev/null; then - SOURCE=$(echo "$RESPONSE" | jq -r '.source') - echo " Source: $SOURCE" - fi - - if echo "$RESPONSE" | jq -e '.is_rdma' > /dev/null; then - IS_RDMA=$(echo "$RESPONSE" | jq -r '.is_rdma') - echo " RDMA Used: $IS_RDMA" - fi -else - echo "โŒ Read endpoint failed!" - echo "Response: $RESPONSE" -fi - -# Stop the sidecar -kill $SIDECAR_PID 2>/dev/null || true -wait $SIDECAR_PID 2>/dev/null || true - -echo "" -echo "๐ŸŽฏ Integration Test Summary:" -echo "==========================" -echo "โœ… Sidecar builds successfully" -echo "โœ… Parse functions handle errors correctly" -echo "โœ… HTTP endpoints are functional" -echo "โœ… JSON responses are properly formatted" -echo "โœ… Error handling works as expected" -echo "" -echo "๐ŸŽ‰ All RDMA integration fixes are working correctly!" -echo "" -echo "๐Ÿ’ก Next Steps:" -echo "- Deploy in Docker environment with real SeaweedFS cluster" -echo "- Test with actual file uploads and downloads" -echo "- Verify RDMA flags are passed correctly to weed mount" -echo "- Monitor health checks with configurable socket paths" diff --git a/seaweedfs-rdma-sidecar/tests/docker-smoke-test.sh b/seaweedfs-rdma-sidecar/tests/docker-smoke-test.sh deleted file mode 100755 index b7ad813c1..000000000 --- a/seaweedfs-rdma-sidecar/tests/docker-smoke-test.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# Simple smoke test for Docker setup -set -e - -echo "๐Ÿงช Docker Smoke Test" -echo "====================" -echo "" - -echo "๐Ÿ“‹ 1. Testing Docker Compose configuration..." -docker-compose config --quiet -echo "โœ… Docker Compose configuration is valid" -echo "" - -echo "๐Ÿ“‹ 2. Testing container builds..." -echo "Building RDMA engine container..." -docker build -f Dockerfile.rdma-engine -t test-rdma-engine . > /dev/null -echo "โœ… RDMA engine container builds successfully" -echo "" - -echo "๐Ÿ“‹ 3. Testing basic container startup..." -echo "Starting RDMA engine container..." -container_id=$(docker run --rm -d --name test-rdma-engine test-rdma-engine) -sleep 5 - -if docker ps | grep test-rdma-engine > /dev/null; then - echo "โœ… RDMA engine container starts successfully" - docker stop test-rdma-engine > /dev/null -else - echo "โŒ RDMA engine container failed to start" - echo "Checking container logs:" - docker logs test-rdma-engine 2>&1 || true - docker stop test-rdma-engine > /dev/null 2>&1 || true - exit 1 -fi -echo "" - -echo "๐ŸŽ‰ All smoke tests passed!" -echo "Docker setup is working correctly." diff --git a/seaweedfs-rdma-sidecar/tests/docker-test-helper.sh b/seaweedfs-rdma-sidecar/tests/docker-test-helper.sh deleted file mode 100755 index edb95541e..000000000 --- a/seaweedfs-rdma-sidecar/tests/docker-test-helper.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash - -# Docker Test Helper - Simplified commands for running integration tests - -set -e - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -print_usage() { - echo -e "${BLUE}SeaweedFS RDMA Docker Integration Test Helper${NC}" - echo "" - echo "Usage: $0 [command]" - echo "" - echo "Commands:" - echo " start - Start all services" - echo " test - Run integration tests" - echo " stop - Stop all services" - echo " clean - Stop services and clean up volumes" - echo " logs - Show logs from all services" - echo " status - Show status of all services" - echo " shell - Open shell in test client container" - echo "" - echo "Examples:" - echo " $0 start # Start all services" - echo " $0 test # Run full integration test suite" - echo " $0 logs rdma-engine # Show logs from RDMA engine" - echo " $0 shell # Interactive testing shell" -} - -start_services() { - echo -e "${GREEN}๐Ÿš€ Starting SeaweedFS RDMA integration services...${NC}" - docker-compose up -d seaweedfs-master seaweedfs-volume rdma-engine rdma-sidecar - - echo -e "${YELLOW}โณ Waiting for services to be ready...${NC}" - sleep 10 - - echo -e "${GREEN}โœ… Services started. Checking health...${NC}" - docker-compose ps -} - -run_tests() { - echo -e "${GREEN}๐Ÿงช Running integration tests...${NC}" - - # Make sure services are running - docker-compose up -d seaweedfs-master seaweedfs-volume rdma-engine rdma-sidecar - - # Wait for services to be ready - echo -e "${YELLOW}โณ Waiting for services to be ready...${NC}" - sleep 15 - - # Run the integration tests - docker-compose run --rm integration-tests -} - -stop_services() { - echo -e "${YELLOW}๐Ÿ›‘ Stopping services...${NC}" - docker-compose down - echo -e "${GREEN}โœ… Services stopped${NC}" -} - -clean_all() { - echo -e "${YELLOW}๐Ÿงน Cleaning up services and volumes...${NC}" - docker-compose down -v --remove-orphans - echo -e "${GREEN}โœ… Cleanup complete${NC}" -} - -show_logs() { - local service=${1:-} - if [ -n "$service" ]; then - echo -e "${BLUE}๐Ÿ“‹ Showing logs for $service...${NC}" - docker-compose logs -f "$service" - else - echo -e "${BLUE}๐Ÿ“‹ Showing logs for all services...${NC}" - docker-compose logs -f - fi -} - -show_status() { - echo -e "${BLUE}๐Ÿ“Š Service Status:${NC}" - docker-compose ps - - echo -e "\n${BLUE}๐Ÿ“ก Health Checks:${NC}" - - # Check SeaweedFS Master - if curl -s http://localhost:9333/cluster/status >/dev/null 2>&1; then - echo -e " ${GREEN}โœ… SeaweedFS Master: Healthy${NC}" - else - echo -e " ${RED}โŒ SeaweedFS Master: Unhealthy${NC}" - fi - - # Check SeaweedFS Volume - if curl -s http://localhost:8080/status >/dev/null 2>&1; then - echo -e " ${GREEN}โœ… SeaweedFS Volume: Healthy${NC}" - else - echo -e " ${RED}โŒ SeaweedFS Volume: Unhealthy${NC}" - fi - - # Check RDMA Sidecar - if curl -s http://localhost:8081/health >/dev/null 2>&1; then - echo -e " ${GREEN}โœ… RDMA Sidecar: Healthy${NC}" - else - echo -e " ${RED}โŒ RDMA Sidecar: Unhealthy${NC}" - fi -} - -open_shell() { - echo -e "${GREEN}๐Ÿš Opening interactive shell in test client...${NC}" - echo -e "${YELLOW}Use './test-rdma --help' for RDMA testing commands${NC}" - echo -e "${YELLOW}Use 'curl http://rdma-sidecar:8081/health' to test sidecar${NC}" - - docker-compose run --rm test-client /bin/bash -} - -# Main command handling -case "${1:-}" in - start) - start_services - ;; - test) - run_tests - ;; - stop) - stop_services - ;; - clean) - clean_all - ;; - logs) - show_logs "${2:-}" - ;; - status) - show_status - ;; - shell) - open_shell - ;; - -h|--help|help) - print_usage - ;; - "") - print_usage - exit 1 - ;; - *) - echo -e "${RED}โŒ Unknown command: $1${NC}" - print_usage - exit 1 - ;; -esac diff --git a/seaweedfs-rdma-sidecar/tests/run-integration-tests.sh b/seaweedfs-rdma-sidecar/tests/run-integration-tests.sh deleted file mode 100755 index 8f23c7e5f..000000000 --- a/seaweedfs-rdma-sidecar/tests/run-integration-tests.sh +++ /dev/null @@ -1,302 +0,0 @@ -#!/bin/bash - -# SeaweedFS RDMA Integration Test Suite -# Comprehensive testing of the complete integration in Docker environment - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -PURPLE='\033[0;35m' -CYAN='\033[0;36m' -NC='\033[0m' # No Color - -print_header() { - echo -e "\n${PURPLE}===============================================${NC}" - echo -e "${PURPLE}$1${NC}" - echo -e "${PURPLE}===============================================${NC}\n" -} - -print_step() { - echo -e "${CYAN}๐Ÿ”ต $1${NC}" -} - -print_success() { - echo -e "${GREEN}โœ… $1${NC}" -} - -print_warning() { - echo -e "${YELLOW}โš ๏ธ $1${NC}" -} - -print_error() { - echo -e "${RED}โŒ $1${NC}" -} - -wait_for_service() { - local url=$1 - local service_name=$2 - local max_attempts=30 - local attempt=1 - - print_step "Waiting for $service_name to be ready..." - - while [ $attempt -le $max_attempts ]; do - if curl -s "$url" > /dev/null 2>&1; then - print_success "$service_name is ready" - return 0 - fi - - echo -n "." - sleep 2 - attempt=$((attempt + 1)) - done - - print_error "$service_name failed to become ready after $max_attempts attempts" - return 1 -} - -test_seaweedfs_master() { - print_header "TESTING SEAWEEDFS MASTER" - - wait_for_service "$SEAWEEDFS_MASTER/cluster/status" "SeaweedFS Master" - - print_step "Checking master status..." - response=$(curl -s "$SEAWEEDFS_MASTER/cluster/status") - - if echo "$response" | jq -e '.IsLeader == true' > /dev/null; then - print_success "SeaweedFS Master is leader and ready" - else - print_error "SeaweedFS Master is not ready" - echo "$response" - return 1 - fi -} - -test_seaweedfs_volume() { - print_header "TESTING SEAWEEDFS VOLUME SERVER" - - wait_for_service "$SEAWEEDFS_VOLUME/status" "SeaweedFS Volume Server" - - print_step "Checking volume server status..." - response=$(curl -s "$SEAWEEDFS_VOLUME/status") - - if echo "$response" | jq -e '.Version' > /dev/null; then - print_success "SeaweedFS Volume Server is ready" - echo "Volume Server Version: $(echo "$response" | jq -r '.Version')" - else - print_error "SeaweedFS Volume Server is not ready" - echo "$response" - return 1 - fi -} - -test_rdma_engine() { - print_header "TESTING RDMA ENGINE" - - print_step "Checking RDMA engine socket..." - if [ -S "$RDMA_SOCKET_PATH" ]; then - print_success "RDMA engine socket exists" - else - print_error "RDMA engine socket not found at $RDMA_SOCKET_PATH" - return 1 - fi - - print_step "Testing RDMA engine ping..." - if ./test-rdma ping --socket "$RDMA_SOCKET_PATH" 2>/dev/null; then - print_success "RDMA engine ping successful" - else - print_error "RDMA engine ping failed" - return 1 - fi - - print_step "Testing RDMA engine capabilities..." - if ./test-rdma capabilities --socket "$RDMA_SOCKET_PATH" 2>/dev/null | grep -q "Version:"; then - print_success "RDMA engine capabilities retrieved" - ./test-rdma capabilities --socket "$RDMA_SOCKET_PATH" 2>/dev/null | head -5 - else - print_error "RDMA engine capabilities failed" - return 1 - fi -} - -test_rdma_sidecar() { - print_header "TESTING RDMA SIDECAR" - - wait_for_service "$SIDECAR_URL/health" "RDMA Sidecar" - - print_step "Testing sidecar health..." - response=$(curl -s "$SIDECAR_URL/health") - - if echo "$response" | jq -e '.status == "healthy"' > /dev/null; then - print_success "RDMA Sidecar is healthy" - echo "RDMA Status: $(echo "$response" | jq -r '.rdma.enabled')" - else - print_error "RDMA Sidecar health check failed" - echo "$response" - return 1 - fi - - print_step "Testing sidecar stats..." - stats=$(curl -s "$SIDECAR_URL/stats") - - if echo "$stats" | jq -e '.enabled' > /dev/null; then - print_success "RDMA Sidecar stats retrieved" - echo "RDMA Enabled: $(echo "$stats" | jq -r '.enabled')" - echo "RDMA Connected: $(echo "$stats" | jq -r '.connected')" - - if echo "$stats" | jq -e '.capabilities' > /dev/null; then - version=$(echo "$stats" | jq -r '.capabilities.version') - sessions=$(echo "$stats" | jq -r '.capabilities.max_sessions') - print_success "RDMA Engine Info: Version=$version, Max Sessions=$sessions" - fi - else - print_error "RDMA Sidecar stats failed" - echo "$stats" - return 1 - fi -} - -test_direct_rdma_operations() { - print_header "TESTING DIRECT RDMA OPERATIONS" - - print_step "Testing direct RDMA read operation..." - if ./test-rdma read --socket "$RDMA_SOCKET_PATH" --volume 1 --needle 12345 --size 1024 2>/dev/null | grep -q "RDMA read completed"; then - print_success "Direct RDMA read operation successful" - else - print_warning "Direct RDMA read operation failed (expected in mock mode)" - fi - - print_step "Running RDMA performance benchmark..." - benchmark_result=$(./test-rdma bench --socket "$RDMA_SOCKET_PATH" --iterations 5 --read-size 2048 2>/dev/null | tail -10) - - if echo "$benchmark_result" | grep -q "Operations/sec:"; then - print_success "RDMA benchmark completed" - echo "$benchmark_result" | grep -E "Operations|Latency|Throughput" - else - print_warning "RDMA benchmark had issues (expected in mock mode)" - fi -} - -test_sidecar_needle_operations() { - print_header "TESTING SIDECAR NEEDLE OPERATIONS" - - print_step "Testing needle read via sidecar..." - response=$(curl -s "$SIDECAR_URL/read?volume=1&needle=12345&cookie=305419896&size=1024") - - if echo "$response" | jq -e '.success == true' > /dev/null; then - print_success "Sidecar needle read successful" - - is_rdma=$(echo "$response" | jq -r '.is_rdma') - source=$(echo "$response" | jq -r '.source') - duration=$(echo "$response" | jq -r '.duration') - - if [ "$is_rdma" = "true" ]; then - print_success "RDMA fast path used! Duration: $duration" - else - print_warning "HTTP fallback used. Duration: $duration" - fi - - echo "Response details:" - echo "$response" | jq '{success, is_rdma, source, duration, data_size}' - else - print_error "Sidecar needle read failed" - echo "$response" - return 1 - fi -} - -test_sidecar_benchmark() { - print_header "TESTING SIDECAR BENCHMARK" - - print_step "Running sidecar performance benchmark..." - response=$(curl -s "$SIDECAR_URL/benchmark?iterations=5&size=2048") - - if echo "$response" | jq -e '.benchmark_results' > /dev/null; then - print_success "Sidecar benchmark completed" - - rdma_ops=$(echo "$response" | jq -r '.benchmark_results.rdma_ops') - http_ops=$(echo "$response" | jq -r '.benchmark_results.http_ops') - avg_latency=$(echo "$response" | jq -r '.benchmark_results.avg_latency') - ops_per_sec=$(echo "$response" | jq -r '.benchmark_results.ops_per_sec') - - echo "Benchmark Results:" - echo " RDMA Operations: $rdma_ops" - echo " HTTP Operations: $http_ops" - echo " Average Latency: $avg_latency" - echo " Operations/sec: $ops_per_sec" - else - print_error "Sidecar benchmark failed" - echo "$response" - return 1 - fi -} - -test_error_handling() { - print_header "TESTING ERROR HANDLING AND FALLBACK" - - print_step "Testing invalid needle read..." - response=$(curl -s "$SIDECAR_URL/read?volume=999&needle=999999&size=1024") - - # Should succeed with mock data or fail gracefully - if echo "$response" | jq -e '.success' > /dev/null; then - result=$(echo "$response" | jq -r '.success') - if [ "$result" = "true" ]; then - print_success "Error handling working - mock data returned" - else - print_success "Error handling working - graceful failure" - fi - else - print_success "Error handling working - proper error response" - fi -} - -main() { - print_header "๐Ÿš€ SEAWEEDFS RDMA INTEGRATION TEST SUITE" - - echo -e "${GREEN}Starting comprehensive integration tests...${NC}" - echo -e "${BLUE}Environment:${NC}" - echo -e " RDMA Socket: $RDMA_SOCKET_PATH" - echo -e " Sidecar URL: $SIDECAR_URL" - echo -e " SeaweedFS Master: $SEAWEEDFS_MASTER" - echo -e " SeaweedFS Volume: $SEAWEEDFS_VOLUME" - - # Run tests in sequence - test_seaweedfs_master - test_seaweedfs_volume - test_rdma_engine - test_rdma_sidecar - test_direct_rdma_operations - test_sidecar_needle_operations - test_sidecar_benchmark - test_error_handling - - print_header "๐ŸŽ‰ ALL INTEGRATION TESTS COMPLETED!" - - echo -e "${GREEN}โœ… Test Summary:${NC}" - echo -e " โœ… SeaweedFS Master: Working" - echo -e " โœ… SeaweedFS Volume Server: Working" - echo -e " โœ… Rust RDMA Engine: Working (Mock Mode)" - echo -e " โœ… Go RDMA Sidecar: Working" - echo -e " โœ… IPC Communication: Working" - echo -e " โœ… Needle Operations: Working" - echo -e " โœ… Performance Benchmarking: Working" - echo -e " โœ… Error Handling: Working" - - print_success "SeaweedFS RDMA integration is fully functional!" - - return 0 -} - -# Check required environment variables -if [ -z "$RDMA_SOCKET_PATH" ] || [ -z "$SIDECAR_URL" ] || [ -z "$SEAWEEDFS_MASTER" ] || [ -z "$SEAWEEDFS_VOLUME" ]; then - print_error "Required environment variables not set" - echo "Required: RDMA_SOCKET_PATH, SIDECAR_URL, SEAWEEDFS_MASTER, SEAWEEDFS_VOLUME" - exit 1 -fi - -# Run main test suite -main "$@" diff --git a/weed/command/mount.go b/weed/command/mount.go index 1cac8613e..22fcd0c8a 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -48,14 +48,6 @@ type MountOptions struct { // Periodic metadata flush to protect against orphan chunk cleanup metadataFlushSeconds *int - // RDMA acceleration options - rdmaEnabled *bool - rdmaSidecarAddr *string - rdmaFallback *bool - rdmaReadOnly *bool - rdmaMaxConcurrent *int - rdmaTimeoutMs *int - // Peer chunk sharing options (design-weed-mount-peer-chunk-sharing.md). peerEnabled *bool peerListen *string @@ -139,14 +131,6 @@ func init() { // Periodic metadata flush to protect against orphan chunk cleanup mountOptions.metadataFlushSeconds = cmdMount.Flag.Int("metadataFlushSeconds", 120, "periodically flush file metadata to filer in seconds (0 to disable). This protects chunks from being purged by volume.fsck for long-running writes") - // RDMA acceleration flags - mountOptions.rdmaEnabled = cmdMount.Flag.Bool("rdma.enabled", false, "enable RDMA acceleration for reads") - mountOptions.rdmaSidecarAddr = cmdMount.Flag.String("rdma.sidecar", "", "RDMA sidecar address (e.g., localhost:8081)") - mountOptions.rdmaFallback = cmdMount.Flag.Bool("rdma.fallback", true, "fallback to HTTP when RDMA fails") - mountOptions.rdmaReadOnly = cmdMount.Flag.Bool("rdma.readOnly", false, "use RDMA for reads only (writes use HTTP)") - mountOptions.rdmaMaxConcurrent = cmdMount.Flag.Int("rdma.maxConcurrent", 64, "max concurrent RDMA operations") - mountOptions.rdmaTimeoutMs = cmdMount.Flag.Int("rdma.timeoutMs", 5000, "RDMA operation timeout in milliseconds") - // Peer chunk sharing flags. mountOptions.peerEnabled = cmdMount.Flag.Bool("peer.enable", false, "opt in to peer chunk sharing โ€” mount serves its chunk cache to other mounts and fetches from peers instead of volume servers when available") mountOptions.peerListen = cmdMount.Flag.String("peer.listen", ":18080", "bind address for peer gRPC (directory RPCs + FetchChunk streaming)") @@ -205,18 +189,5 @@ var cmdMount = &Command{ -dir=\\seaweedfs\Images labels the disk "Images" while still mounting everything, and only a bare drive letter falls back to the filer address. - RDMA Acceleration: - For ultra-fast reads, enable RDMA acceleration with an RDMA sidecar: - weed mount -filer=localhost:8888 -dir=/mnt/seaweedfs \ - -rdma.enabled=true -rdma.sidecar=localhost:8081 - - RDMA Options: - -rdma.enabled=false Enable RDMA acceleration for reads - -rdma.sidecar="" RDMA sidecar address (required if enabled) - -rdma.fallback=true Fallback to HTTP when RDMA fails - -rdma.readOnly=false Use RDMA for reads only (writes use HTTP) - -rdma.maxConcurrent=64 Max concurrent RDMA operations - -rdma.timeoutMs=5000 RDMA operation timeout in milliseconds - `, } diff --git a/weed/command/mount_common.go b/weed/command/mount_common.go index 4c2ce7e2a..bdef9e83c 100644 --- a/weed/command/mount_common.go +++ b/weed/command/mount_common.go @@ -221,18 +221,11 @@ func buildSeaweedFileSystem(option *MountOptions, p fileSystemParams) *mount.WFS DisableXAttr: *option.disableXAttr, IsMacOs: runtime.GOOS == "darwin", MetadataFlushSeconds: *option.metadataFlushSeconds, - // RDMA acceleration options - RdmaEnabled: *option.rdmaEnabled, - RdmaSidecarAddr: *option.rdmaSidecarAddr, - RdmaFallback: *option.rdmaFallback, - RdmaReadOnly: *option.rdmaReadOnly, - RdmaMaxConcurrent: *option.rdmaMaxConcurrent, - RdmaTimeoutMs: *option.rdmaTimeoutMs, - DirIdleEvictSec: *option.dirIdleEvictSec, - EnableDistributedLock: option.distributedLock != nil && *option.distributedLock, - WritebackCache: option.writebackCache != nil && *option.writebackCache, - EagerFilerCreate: p.eagerFilerCreate, - PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink, + DirIdleEvictSec: *option.dirIdleEvictSec, + EnableDistributedLock: option.distributedLock != nil && *option.distributedLock, + WritebackCache: option.writebackCache != nil && *option.writebackCache, + EagerFilerCreate: p.eagerFilerCreate, + PosixDirNlink: option.posixDirNlink != nil && *option.posixDirNlink, // Peer chunk sharing PeerEnabled: option.peerEnabled != nil && *option.peerEnabled, PeerListen: peerStringOrEmpty(option.peerListen), diff --git a/weed/mount/filehandle.go b/weed/mount/filehandle.go index a2c5560d4..2526bce32 100644 --- a/weed/mount/filehandle.go +++ b/weed/mount/filehandle.go @@ -71,11 +71,6 @@ type FileHandle struct { // only the handle's shared lock and so races a second concurrent read. remoteInstallMu sync.Mutex - // RDMA chunk offset cache for performance optimization - chunkOffsetCache []int64 - chunkCacheValid bool - chunkCacheLock sync.RWMutex - // for debugging mirrorFile *os.File } @@ -142,9 +137,6 @@ func (fh *FileHandle) SetEntry(entry *filer_pb.Entry) { glog.Fatalf("setting file handle entry to nil") } fh.entry.SetEntry(entry) - - // Invalidate chunk offset cache since chunks may have changed - fh.invalidateChunkCache() } // installAckedEntry installs filer-acknowledged state under the handle lock @@ -208,17 +200,11 @@ func (fh *FileHandle) ResetDirtyPages() { func (fh *FileHandle) UpdateEntry(fn func(entry *filer_pb.Entry)) *filer_pb.Entry { result := fh.entry.UpdateEntry(fn) - // Invalidate chunk offset cache since entry may have been modified - fh.invalidateChunkCache() - return result } func (fh *FileHandle) AddChunks(chunks []*filer_pb.FileChunk) { fh.entry.AppendChunks(chunks) - - // Invalidate chunk offset cache since new chunks were added - fh.invalidateChunkCache() } func (fh *FileHandle) ReleaseHandle() { @@ -238,48 +224,3 @@ func (fh *FileHandle) ReleaseHandle() { fh.mirrorFile.Close() } } - -// getCumulativeOffsets returns cached cumulative offsets for chunks, computing them if necessary -func (fh *FileHandle) getCumulativeOffsets(chunks []*filer_pb.FileChunk) []int64 { - fh.chunkCacheLock.RLock() - if fh.chunkCacheValid && len(fh.chunkOffsetCache) == len(chunks)+1 { - // Cache is valid and matches current chunk count - result := make([]int64, len(fh.chunkOffsetCache)) - copy(result, fh.chunkOffsetCache) - fh.chunkCacheLock.RUnlock() - return result - } - fh.chunkCacheLock.RUnlock() - - // Need to compute/recompute cache - fh.chunkCacheLock.Lock() - defer fh.chunkCacheLock.Unlock() - - // Double-check in case another goroutine computed it while we waited for the lock - if fh.chunkCacheValid && len(fh.chunkOffsetCache) == len(chunks)+1 { - result := make([]int64, len(fh.chunkOffsetCache)) - copy(result, fh.chunkOffsetCache) - return result - } - - // Compute cumulative offsets - cumulativeOffsets := make([]int64, len(chunks)+1) - for i, chunk := range chunks { - cumulativeOffsets[i+1] = cumulativeOffsets[i] + int64(chunk.Size) - } - - // Cache the result - fh.chunkOffsetCache = make([]int64, len(cumulativeOffsets)) - copy(fh.chunkOffsetCache, cumulativeOffsets) - fh.chunkCacheValid = true - - return cumulativeOffsets -} - -// invalidateChunkCache invalidates the chunk offset cache when chunks are modified -func (fh *FileHandle) invalidateChunkCache() { - fh.chunkCacheLock.Lock() - fh.chunkCacheValid = false - fh.chunkOffsetCache = nil - fh.chunkCacheLock.Unlock() -} diff --git a/weed/mount/filehandle_read.go b/weed/mount/filehandle_read.go index b804dab22..014b1947c 100644 --- a/weed/mount/filehandle_read.go +++ b/weed/mount/filehandle_read.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "io" - "sort" "google.golang.org/protobuf/proto" @@ -83,16 +82,6 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte return int64(totalRead), 0, nil } - // Try RDMA acceleration first if available - if fh.wfs.rdmaClient != nil && fh.wfs.option.RdmaEnabled { - totalRead, ts, err := fh.tryRDMARead(ctx, fileSize, buff, offset, chunks) - if err == nil { - glog.V(4).Infof("RDMA read successful for %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead) - return int64(totalRead), ts, nil - } - glog.V(4).Infof("RDMA read failed for %s, falling back to HTTP: %v", fileFullPath, err) - } - // Peer chunk sharing: try a peer mount's cache before the volume tier. // Any failure falls through transparently. See design-weed-mount- // peer-chunk-sharing.md ยง4.3. @@ -122,61 +111,6 @@ func (fh *FileHandle) readFromChunksWithContext(ctx context.Context, buff []byte return int64(totalRead), ts, err } -// tryRDMARead attempts to read file data using RDMA acceleration. chunks is a -// snapshot captured under the LockedEntry lock by the caller. -func (fh *FileHandle) tryRDMARead(ctx context.Context, fileSize int64, buff []byte, offset int64, chunks []*filer_pb.FileChunk) (int64, int64, error) { - // For now, we'll try to read the chunks directly using RDMA - // This is a simplified approach - in a full implementation, we'd need to - // handle chunk boundaries, multiple chunks, etc. - - if len(chunks) == 0 { - return 0, 0, fmt.Errorf("no chunks available for RDMA read") - } - - // Find the chunk that contains our offset using binary search - var targetChunk *filer_pb.FileChunk - var chunkOffset int64 - - // Get cached cumulative offsets for efficient binary search - cumulativeOffsets := fh.getCumulativeOffsets(chunks) - - // Use binary search to find the chunk containing the offset - chunkIndex := sort.Search(len(chunks), func(i int) bool { - return offset < cumulativeOffsets[i+1] - }) - - // Verify the chunk actually contains our offset - if chunkIndex < len(chunks) && offset >= cumulativeOffsets[chunkIndex] { - targetChunk = chunks[chunkIndex] - chunkOffset = offset - cumulativeOffsets[chunkIndex] - } - - if targetChunk == nil { - return 0, 0, fmt.Errorf("no chunk found for offset %d", offset) - } - - // Calculate how much to read from this chunk - remainingInChunk := int64(targetChunk.Size) - chunkOffset - readSize := min(int64(len(buff)), remainingInChunk) - - glog.V(4).Infof("RDMA read attempt: chunk=%s (fileId=%s), chunkOffset=%d, readSize=%d", - targetChunk.FileId, targetChunk.FileId, chunkOffset, readSize) - - // Try RDMA read using file ID directly (more efficient) - data, isRDMA, err := fh.wfs.rdmaClient.ReadNeedle(ctx, targetChunk.FileId, uint64(chunkOffset), uint64(readSize)) - if err != nil { - return 0, 0, fmt.Errorf("RDMA read failed: %w", err) - } - - if !isRDMA { - return 0, 0, fmt.Errorf("RDMA not available for chunk") - } - - // Copy data to buffer - copied := copy(buff, data) - return int64(copied), targetChunk.ModifiedTsNs, nil -} - func (fh *FileHandle) downloadRemoteEntry(entry *LockedEntry) error { fileFullPath := fh.FullPath() diff --git a/weed/mount/rdma_client.go b/weed/mount/rdma_client.go deleted file mode 100644 index 6a77f8f52..000000000 --- a/weed/mount/rdma_client.go +++ /dev/null @@ -1,373 +0,0 @@ -package mount - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strings" - "sync/atomic" - "time" - - "github.com/seaweedfs/seaweedfs/weed/glog" - "github.com/seaweedfs/seaweedfs/weed/wdclient" -) - -// RDMAMountClient provides RDMA acceleration for SeaweedFS mount operations -type RDMAMountClient struct { - sidecarAddr string - httpClient *http.Client - maxConcurrent int - timeout time.Duration - semaphore chan struct{} - - // Volume lookup - lookupFileIdFn wdclient.LookupFileIdFunctionType - - // Statistics - totalRequests atomic.Int64 - successfulReads atomic.Int64 - failedReads atomic.Int64 - totalBytesRead atomic.Int64 - totalLatencyNs atomic.Int64 -} - -// RDMAReadRequest represents a request to read data via RDMA -type RDMAReadRequest struct { - VolumeID uint32 `json:"volume_id"` - NeedleID uint64 `json:"needle_id"` - Cookie uint32 `json:"cookie"` - Offset uint64 `json:"offset"` - Size uint64 `json:"size"` -} - -// RDMAReadResponse represents the response from an RDMA read operation -type RDMAReadResponse struct { - Success bool `json:"success"` - IsRDMA bool `json:"is_rdma"` - Source string `json:"source"` - Duration string `json:"duration"` - DataSize int `json:"data_size"` - SessionID string `json:"session_id,omitempty"` - ErrorMsg string `json:"error,omitempty"` - - // Zero-copy optimization fields - UseTempFile bool `json:"use_temp_file"` - TempFile string `json:"temp_file"` -} - -// RDMAHealthResponse represents the health status of the RDMA sidecar -type RDMAHealthResponse struct { - Status string `json:"status"` - RDMA struct { - Enabled bool `json:"enabled"` - Connected bool `json:"connected"` - } `json:"rdma"` - Timestamp string `json:"timestamp"` -} - -// NewRDMAMountClient creates a new RDMA client for mount operations -func NewRDMAMountClient(sidecarAddr string, lookupFileIdFn wdclient.LookupFileIdFunctionType, maxConcurrent int, timeoutMs int) (*RDMAMountClient, error) { - client := &RDMAMountClient{ - sidecarAddr: sidecarAddr, - maxConcurrent: maxConcurrent, - timeout: time.Duration(timeoutMs) * time.Millisecond, - httpClient: &http.Client{ - Timeout: time.Duration(timeoutMs) * time.Millisecond, - }, - semaphore: make(chan struct{}, maxConcurrent), - lookupFileIdFn: lookupFileIdFn, - } - - // Test connectivity and RDMA availability - if err := client.healthCheck(); err != nil { - return nil, fmt.Errorf("RDMA sidecar health check failed: %w", err) - } - - glog.Infof("RDMA mount client initialized: sidecar=%s, maxConcurrent=%d, timeout=%v", - sidecarAddr, maxConcurrent, client.timeout) - - return client, nil -} - -// lookupVolumeLocationByFileID finds the best volume server for a given file ID -func (c *RDMAMountClient) lookupVolumeLocationByFileID(ctx context.Context, fileID string) (string, error) { - glog.V(4).Infof("Looking up volume location for file ID %s", fileID) - - targetUrls, err := c.lookupFileIdFn(ctx, fileID) - if err != nil { - return "", fmt.Errorf("failed to lookup volume for file %s: %w", fileID, err) - } - - if len(targetUrls) == 0 { - return "", fmt.Errorf("no locations found for file %s", fileID) - } - - // Choose the first URL and extract the server address - targetUrl := targetUrls[0] - // Extract server address from URL like "http://server:port/fileId" - parts := strings.Split(targetUrl, "/") - if len(parts) < 3 { - return "", fmt.Errorf("invalid target URL format: %s", targetUrl) - } - bestAddress := fmt.Sprintf("http://%s", parts[2]) - - glog.V(4).Infof("File %s located at %s", fileID, bestAddress) - return bestAddress, nil -} - -// healthCheck verifies that the RDMA sidecar is available and functioning -func (c *RDMAMountClient) healthCheck() error { - ctx, cancel := context.WithTimeout(context.Background(), c.timeout) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, "GET", - fmt.Sprintf("http://%s/health", c.sidecarAddr), nil) - if err != nil { - return fmt.Errorf("failed to create health check request: %w", err) - } - - resp, err := c.httpClient.Do(req) - if err != nil { - return fmt.Errorf("health check request failed: %w", err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("health check failed with status: %s", resp.Status) - } - - // Parse health response - var health RDMAHealthResponse - if err := json.NewDecoder(resp.Body).Decode(&health); err != nil { - return fmt.Errorf("failed to parse health response: %w", err) - } - - if health.Status != "healthy" { - return fmt.Errorf("sidecar reports unhealthy status: %s", health.Status) - } - - if !health.RDMA.Enabled { - return fmt.Errorf("RDMA is not enabled on sidecar") - } - - if !health.RDMA.Connected { - glog.Warningf("RDMA sidecar is healthy but not connected to RDMA engine") - } - - return nil -} - -// ReadNeedle reads data from a specific needle using RDMA acceleration -func (c *RDMAMountClient) ReadNeedle(ctx context.Context, fileID string, offset, size uint64) ([]byte, bool, error) { - // Acquire semaphore for concurrency control - select { - case c.semaphore <- struct{}{}: - defer func() { <-c.semaphore }() - case <-ctx.Done(): - return nil, false, ctx.Err() - } - - c.totalRequests.Add(1) - startTime := time.Now() - - // Lookup volume location using file ID directly - volumeServer, err := c.lookupVolumeLocationByFileID(ctx, fileID) - if err != nil { - c.failedReads.Add(1) - return nil, false, fmt.Errorf("failed to lookup volume for file %s: %w", fileID, err) - } - - // Prepare request URL with file_id parameter (simpler than individual components) - reqURL := fmt.Sprintf("http://%s/read?file_id=%s&offset=%d&size=%d&volume_server=%s", - c.sidecarAddr, fileID, offset, size, volumeServer) - - req, err := http.NewRequestWithContext(ctx, "GET", reqURL, nil) - if err != nil { - c.failedReads.Add(1) - return nil, false, fmt.Errorf("failed to create RDMA request: %w", err) - } - - // Execute request - resp, err := c.httpClient.Do(req) - if err != nil { - c.failedReads.Add(1) - return nil, false, fmt.Errorf("RDMA request failed: %w", err) - } - defer resp.Body.Close() - - duration := time.Since(startTime) - c.totalLatencyNs.Add(duration.Nanoseconds()) - - if resp.StatusCode != http.StatusOK { - c.failedReads.Add(1) - body, _ := io.ReadAll(resp.Body) - return nil, false, fmt.Errorf("RDMA read failed with status %s: %s", resp.Status, string(body)) - } - - // Check if response indicates RDMA was used - contentType := resp.Header.Get("Content-Type") - isRDMA := strings.Contains(resp.Header.Get("X-Source"), "rdma") || - resp.Header.Get("X-RDMA-Used") == "true" - - // Check for zero-copy temp file optimization - tempFilePath := resp.Header.Get("X-Temp-File") - useTempFile := resp.Header.Get("X-Use-Temp-File") == "true" - - var data []byte - - var n int - if useTempFile && tempFilePath != "" { - // Zero-copy path: read from temp file (page cache) - glog.V(4).Infof("๐Ÿ”ฅ Using zero-copy temp file: %s", tempFilePath) - - // Allocate buffer for temp file read - var bufferSize uint64 = 1024 * 1024 // Default 1MB - if size > 0 { - bufferSize = size - } - buffer := make([]byte, bufferSize) - - n, err = c.readFromTempFile(tempFilePath, buffer) - if err != nil { - glog.V(2).Infof("Zero-copy failed, falling back to HTTP body: %v", err) - // Fall back to reading HTTP body - data, err = io.ReadAll(resp.Body) - } else { - data = buffer[:n] - glog.V(4).Infof("๐Ÿ”ฅ Zero-copy successful: %d bytes from page cache", n) - } - - // Important: Cleanup temp file after reading (consumer responsibility) - // This prevents accumulation of temp files in /tmp/rdma-cache - go c.cleanupTempFile(tempFilePath) - } else { - // Regular path: read from HTTP response body - data, err = io.ReadAll(resp.Body) - } - - if err != nil { - c.failedReads.Add(1) - return nil, false, fmt.Errorf("failed to read RDMA response: %w", err) - } - - c.successfulReads.Add(1) - c.totalBytesRead.Add(int64(len(data))) - - // Log successful operation - glog.V(4).Infof("RDMA read completed: fileID=%s, size=%d, duration=%v, rdma=%v, contentType=%s", - fileID, size, duration, isRDMA, contentType) - - return data, isRDMA, nil -} - -// cleanupTempFile requests cleanup of a temp file from the sidecar -func (c *RDMAMountClient) cleanupTempFile(tempFilePath string) { - if tempFilePath == "" { - return - } - - // Give the page cache a brief moment to be utilized before cleanup - // This preserves the zero-copy performance window - time.Sleep(100 * time.Millisecond) - - // Call sidecar cleanup endpoint - cleanupURL := fmt.Sprintf("http://%s/cleanup?temp_file=%s", c.sidecarAddr, url.QueryEscape(tempFilePath)) - - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, "DELETE", cleanupURL, nil) - if err != nil { - glog.V(2).Infof("Failed to create cleanup request for %s: %v", tempFilePath, err) - return - } - - resp, err := c.httpClient.Do(req) - if err != nil { - glog.V(2).Infof("Failed to cleanup temp file %s: %v", tempFilePath, err) - return - } - defer resp.Body.Close() - - if resp.StatusCode == http.StatusOK { - glog.V(4).Infof("๐Ÿงน Temp file cleaned up: %s", tempFilePath) - } else { - glog.V(2).Infof("Cleanup failed for %s: status %s", tempFilePath, resp.Status) - } -} - -// GetStats returns current RDMA client statistics -func (c *RDMAMountClient) GetStats() map[string]interface{} { - totalRequests := c.totalRequests.Load() - successfulReads := c.successfulReads.Load() - failedReads := c.failedReads.Load() - totalBytesRead := c.totalBytesRead.Load() - totalLatencyNs := c.totalLatencyNs.Load() - - successRate := float64(0) - avgLatencyNs := int64(0) - - if totalRequests > 0 { - successRate = float64(successfulReads) / float64(totalRequests) * 100 - avgLatencyNs = totalLatencyNs / totalRequests - } - - return map[string]interface{}{ - "sidecar_addr": c.sidecarAddr, - "max_concurrent": c.maxConcurrent, - "timeout_ms": int(c.timeout / time.Millisecond), - "total_requests": totalRequests, - "successful_reads": successfulReads, - "failed_reads": failedReads, - "success_rate_pct": fmt.Sprintf("%.1f", successRate), - "total_bytes_read": totalBytesRead, - "avg_latency_ns": avgLatencyNs, - "avg_latency_ms": fmt.Sprintf("%.3f", float64(avgLatencyNs)/1000000), - } -} - -// Close shuts down the RDMA client and releases resources -func (c *RDMAMountClient) Close() error { - // No need to close semaphore channel; closing it may cause panics if goroutines are still using it. - // The semaphore will be garbage collected when the client is no longer referenced. - - // Log final statistics - stats := c.GetStats() - glog.Infof("RDMA mount client closing: %+v", stats) - - return nil -} - -// IsHealthy checks if the RDMA sidecar is currently healthy -func (c *RDMAMountClient) IsHealthy() bool { - err := c.healthCheck() - return err == nil -} - -// readFromTempFile performs zero-copy read from temp file using page cache -func (c *RDMAMountClient) readFromTempFile(tempFilePath string, buffer []byte) (int, error) { - if tempFilePath == "" { - return 0, fmt.Errorf("empty temp file path") - } - - // Open temp file for reading - file, err := os.Open(tempFilePath) - if err != nil { - return 0, fmt.Errorf("failed to open temp file %s: %w", tempFilePath, err) - } - defer file.Close() - - // Read from temp file (this should be served from page cache) - n, err := file.Read(buffer) - if err != nil && err != io.EOF { - return n, fmt.Errorf("failed to read from temp file: %w", err) - } - - glog.V(4).Infof("๐Ÿ”ฅ Zero-copy read: %d bytes from temp file %s", n, tempFilePath) - - return n, nil -} diff --git a/weed/mount/weedfs.go b/weed/mount/weedfs.go index 4039c5ba3..b915d6a7b 100644 --- a/weed/mount/weedfs.go +++ b/weed/mount/weedfs.go @@ -87,14 +87,6 @@ type Option struct { // This protects chunks from being purged by volume.fsck for long-running writes MetadataFlushSeconds int - // RDMA acceleration options - RdmaEnabled bool - RdmaSidecarAddr string - RdmaFallback bool - RdmaReadOnly bool - RdmaMaxConcurrent int - RdmaTimeoutMs int - // Peer chunk sharing options (design-weed-mount-peer-chunk-sharing.md). // When PeerEnabled is false (default), the mount runs exactly as today. // One gRPC port carries everything: directory RPCs (ChunkAnnounce / @@ -162,7 +154,6 @@ type WFS struct { posixSid uint64 // this mount's session id, for routed-lock owner identity posixHint *posixLockHint // local fcntl-lock hint for routed mode posixOwn *posixlock.Manager // mirror of locks this mount holds, re-asserted via keepalive - rdmaClient *RDMAMountClient peerRegistrar *PeerRegistrar peerDirectory *PeerDirectory peerGrpcServer *PeerGrpcServer @@ -340,9 +331,6 @@ func NewSeaweedFileSystem(option *Option) *WFS { os.RemoveAll(option.getUniqueCacheDirForWrite()) } os.RemoveAll(option.getUniqueCacheDirForRead()) - if wfs.rdmaClient != nil { - wfs.rdmaClient.Close() - } if wfs.peerAnnouncer != nil { wfs.peerAnnouncer.Stop() } @@ -365,23 +353,6 @@ func NewSeaweedFileSystem(option *Option) *WFS { } }) - // Initialize RDMA client if enabled - if option.RdmaEnabled && option.RdmaSidecarAddr != "" { - rdmaClient, err := NewRDMAMountClient( - option.RdmaSidecarAddr, - wfs.LookupFn(), - option.RdmaMaxConcurrent, - option.RdmaTimeoutMs, - ) - if err != nil { - glog.Warningf("Failed to initialize RDMA client: %v", err) - } else { - wfs.rdmaClient = rdmaClient - glog.Infof("RDMA acceleration enabled: sidecar=%s, maxConcurrent=%d, timeout=%dms", - option.RdmaSidecarAddr, option.RdmaMaxConcurrent, option.RdmaTimeoutMs) - } - } - // Peer chunk sharing: register with every configured filer's mount // registry + start the single gRPC server that handles ChunkAnnounce / // ChunkLookup / FetchChunk. Broadcasting registration to the full