feat: Add multi-threaded file transfers and enhanced logging support

- Implement concurrent file transfer processing with configurable concurrency
- Add new `max_concurrent_transfers` column to transfer configurations
- Enhance scheduler to support multi-threaded file transfers
- Introduce advanced logging system with rotation and configurable log levels
- Update Docker Compose and documentation with new logging configuration options
- Modify directory structure to separate data, backups, and logs
- Add environment variables for comprehensive logging control
- Improve error handling and logging in file transfer processes
This commit is contained in:
StarFleetCPTN committed 2025-03-11 18:00:10 -07:00
1 parent 16b234b2d3
commit ed81b2c9be
13 files changed
+693 -329

No files matched your search

+4 -1
View File
@@ -73,6 +73,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled := false
deleteAfterTransfer := false
skipProcessedFiles := true
maxConcurrentTransfers := 4
rcloneFlags := ""
// If editing an existing config, populate with those values
@@ -131,6 +132,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled = config.ArchiveEnabled
deleteAfterTransfer = config.DeleteAfterTransfer
skipProcessedFiles = config.SkipProcessedFiles
maxConcurrentTransfers = config.MaxConcurrentTransfers
rcloneFlags = config.RcloneFlags
}
@@ -186,6 +188,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled: %v,
deleteAfterTransfer: %v,
skipProcessedFiles: %v,
maxConcurrentTransfers: %d,
rcloneFlags: '%s',
}`,
name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
@@ -195,7 +198,7 @@ func getInitialData(config *db.TransferConfig) string {
destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
destClientId, destClientSecret, destDriveId, destTeamDrive,
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, rcloneFlags)
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
}
templ ConfigForm(ctx context.Context, data ConfigFormData) {