feat: Add multi-threaded file transfers and enhanced logging support

- Implement concurrent file transfer processing with configurable concurrency
- Add new `max_concurrent_transfers` column to transfer configurations
- Enhance scheduler to support multi-threaded file transfers
- Introduce advanced logging system with rotation and configurable log levels
- Update Docker Compose and documentation with new logging configuration options
- Modify directory structure to separate data, backups, and logs
- Add environment variables for comprehensive logging control
- Improve error handling and logging in file transfer processes
This commit is contained in:
StarFleetCPTN
2025-03-11 18:00:10 -07:00
parent 16b234b2d3
commit ed81b2c9be
13 changed files with 693 additions and 329 deletions
+4 -1
View File
@@ -73,6 +73,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled := false
deleteAfterTransfer := false
skipProcessedFiles := true
maxConcurrentTransfers := 4
rcloneFlags := ""
// If editing an existing config, populate with those values
@@ -131,6 +132,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled = config.ArchiveEnabled
deleteAfterTransfer = config.DeleteAfterTransfer
skipProcessedFiles = config.SkipProcessedFiles
maxConcurrentTransfers = config.MaxConcurrentTransfers
rcloneFlags = config.RcloneFlags
}
@@ -186,6 +188,7 @@ func getInitialData(config *db.TransferConfig) string {
archiveEnabled: %v,
deleteAfterTransfer: %v,
skipProcessedFiles: %v,
maxConcurrentTransfers: %d,
rcloneFlags: '%s',
}`,
name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
@@ -195,7 +198,7 @@ func getInitialData(config *db.TransferConfig) string {
destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
destClientId, destClientSecret, destDriveId, destTeamDrive,
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, rcloneFlags)
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
}
templ ConfigForm(ctx context.Context, data ConfigFormData) {
+15
View File
@@ -141,6 +141,21 @@ templ ArchiveOptions() {
Files with the same hash that have been successfully processed before will be skipped
</p>
</div>
<div class="mb-4">
<label for="max_concurrent_transfers" class="block text-sm font-medium text-gray-700 mb-1">Concurrent Transfers</label>
<div class="flex items-center">
<input type="range" id="max_concurrent_transfers" name="max_concurrent_transfers"
class="form-range w-2/3 h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer"
min="1" max="20" step="1"
x-model="maxConcurrentTransfers"
/>
<span class="ml-4 text-gray-700" x-text="maxConcurrentTransfers"></span>
</div>
<p class="mt-1 text-xs text-gray-500">
Number of files to transfer simultaneously (higher values may improve performance but increase resource usage)
</p>
</div>
</div>
}