feat: Add multi-threaded file transfers and enhanced logging support

- Implement concurrent file transfer processing with configurable concurrency
- Add new `max_concurrent_transfers` column to transfer configurations
- Enhance scheduler to support multi-threaded file transfers
- Introduce advanced logging system with rotation and configurable log levels
- Update Docker Compose and documentation with new logging configuration options
- Modify directory structure to separate data, backups, and logs
- Add environment variables for comprehensive logging control
- Improve error handling and logging in file transfer processes

internal/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -35,8 +34,8 @@ func Load() (*Config, error) {
 	// Default configuration
 	cfg := &Config{
 		ServerAddress: ":8080",
-		DataDir:       filepath.Join("./data", "gomft"),
-		BackupDir:     filepath.Join("./data", "gomft", "backups"),
+		DataDir:       "./data",
+		BackupDir:     "./backups",
 		JWTSecret:     "change_this_to_a_secure_random_string",
 		BaseURL:       "http://localhost:8080",
 		Email: EmailConfig{

internal/db/db.go

@@ -98,15 +98,16 @@ type TransferConfig struct {
 	DestDriveID      string `form:"dest_drive_id"`               // For OneDrive
 	DestTeamDrive    string `form:"dest_team_drive"`             // For Google Drive
 	// General fields
-	ArchivePath         string `form:"archive_path"`
-	ArchiveEnabled      bool   `gorm:"default:false" form:"archive_enabled"`
-	RcloneFlags         string `form:"rclone_flags"`
-	DeleteAfterTransfer bool   `gorm:"default:false" form:"delete_after_transfer"`
-	SkipProcessedFiles  bool   `gorm:"default:true" form:"skip_processed_files"`
-	CreatedBy           uint
-	User                User `gorm:"foreignkey:CreatedBy"`
-	CreatedAt           time.Time
-	UpdatedAt           time.Time
+	ArchivePath            string `form:"archive_path"`
+	ArchiveEnabled         bool   `gorm:"default:false" form:"archive_enabled"`
+	RcloneFlags            string `form:"rclone_flags"`
+	DeleteAfterTransfer    bool   `gorm:"default:false" form:"delete_after_transfer"`
+	SkipProcessedFiles     bool   `gorm:"default:true" form:"skip_processed_files"`
+	MaxConcurrentTransfers int    `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers
+	CreatedBy              uint
+	User                   User `gorm:"foreignkey:CreatedBy"`
+	CreatedAt              time.Time
+	UpdatedAt              time.Time
 }
 
 type Job struct {
@@ -381,7 +382,14 @@ func (db *DB) DeleteFileMetadata(id uint) error {
 
 // GetConfigRclonePath returns the path to the rclone config file for a given transfer config
 func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
-	return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
+	// Get data directory from environment or use default
+	dataDir := os.Getenv("DATA_DIR")
+	if dataDir == "" {
+		dataDir = "./data"
+	}
+
+	// Store configs in the data directory
+	return filepath.Join(dataDir, "configs", fmt.Sprintf("config_%d.conf", config.ID))
 }
 
 // GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
@@ -397,8 +405,11 @@ func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
 func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 	configPath := db.GetConfigRclonePath(config)
 
+	// Get the directory part of the path
+	configDir := filepath.Dir(configPath)
+
 	// Ensure configs directory exists
-	if err := os.MkdirAll("configs", 0755); err != nil {
+	if err := os.MkdirAll(configDir, 0755); err != nil {
 		return fmt.Errorf("failed to create configs directory: %v", err)
 	}
 
@@ -792,3 +803,9 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 
 	return nil
 }
+
+func (db *DB) GetActiveJobs() ([]Job, error) {
+	var jobs []Job
+	err := db.Preload("Config").Where("enabled = ?", true).Find(&jobs).Error
+	return jobs, err
+}

internal/db/migrations/add_max_concurrent_transfers.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddMaxConcurrentTransfersColumn adds the max_concurrent_transfers column to transfer_configs table
+func AddMaxConcurrentTransfersColumn() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "20250311_add_max_concurrent_transfers",
+		Migrate: func(tx *gorm.DB) error {
+			// Add max_concurrent_transfers column with default value of 4
+			return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN max_concurrent_transfers INTEGER DEFAULT 4").Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the column if needed
+			return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN max_concurrent_transfers").Error
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -12,6 +12,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		AddDeleteAfterTransferColumn(),
 		AddCloudStorageFields(),
 		AddSkipProcessedFilesColumn(),
+		AddMaxConcurrentTransfersColumn(),
 	}
 
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)