feat: Add multi-threaded file transfers and enhanced logging support

- Implement concurrent file transfer processing with configurable concurrency
- Add new `max_concurrent_transfers` column to transfer configurations
- Enhance scheduler to support multi-threaded file transfers
- Introduce advanced logging system with rotation and configurable log levels
- Update Docker Compose and documentation with new logging configuration options
- Modify directory structure to separate data, backups, and logs
- Add environment variables for comprehensive logging control
- Improve error handling and logging in file transfer processes

internal/db/migrations/add_max_concurrent_transfers.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddMaxConcurrentTransfersColumn adds the max_concurrent_transfers column to transfer_configs table
+func AddMaxConcurrentTransfersColumn() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "20250311_add_max_concurrent_transfers",
+		Migrate: func(tx *gorm.DB) error {
+			// Add max_concurrent_transfers column with default value of 4
+			return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN max_concurrent_transfers INTEGER DEFAULT 4").Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the column if needed
+			return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN max_concurrent_transfers").Error
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -12,6 +12,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		AddDeleteAfterTransferColumn(),
 		AddCloudStorageFields(),
 		AddSkipProcessedFilesColumn(),
+		AddMaxConcurrentTransfersColumn(),
 	}
 
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)