feat: Add multi-threaded file transfers and enhanced logging support

- Implement concurrent file transfer processing with configurable concurrency
- Add new `max_concurrent_transfers` column to transfer configurations
- Enhance scheduler to support multi-threaded file transfers
- Introduce advanced logging system with rotation and configurable log levels
- Update Docker Compose and documentation with new logging configuration options
- Modify directory structure to separate data, backups, and logs
- Add environment variables for comprehensive logging control
- Improve error handling and logging in file transfer processes

.gitignore

@@ -68,11 +68,14 @@ tmp/
 # Ignore the configs directory
 configs/
 
+# Ignore the backups directory
+backups/
+
 # Ignore Dirs
 /source/
 /destination/
-
 /archive/
 
+
 # Ignore binaries
 gomft

Dockerfile

@@ -47,8 +47,8 @@ COPY --from=builder /usr/local/bin/rclone /usr/local/bin/rclone
 COPY static/ /app/static/
 COPY components/ /app/components/
 
-# Create data directory
-RUN mkdir -p /app/data/gomft
+# Create data and backup directories
+RUN mkdir -p /app/data /app/backups
 
 # Set executable permissions
 RUN chmod +x /app/gomft

README.md

@@ -52,6 +52,12 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging
   - Advanced search interface with multiple criteria
   - Bulk management and record deletion capabilities
   - Responsive design with mobile-friendly interface
+- **Multi-threaded File Transfers**: Significantly improve performance with concurrent file processing:
+  - Configurable number of concurrent transfers (1-32) per job
+  - Automatic queue management to prevent system overload
+  - Independent configuration for each transfer job
+  - Optimized for both high-volume small files and large file transfers
+  - Maximizes bandwidth utilization for cloud storage providers
 - **Web Interface**: User-friendly interface for managing transfers, built with Templ components
 - **File Pattern Matching**: Support for file patterns to filter files during transfers
 - **File Output Patterns**: Dynamic naming of destination files using patterns with date variables
@@ -106,6 +112,7 @@ docker run -d \
   --name gomft \
   -p 8080:8080 \
   -v /path/to/data:/app/data \
+  -v /path/to/backups:/app/backups \
   starfleetcptn/gomft:latest
 ```
 
@@ -126,12 +133,12 @@ services:
       - "8080:8080"
     volumes:
       - ./data:/app/data
-      - ./backups:/app/data/gomft/backups
+      - ./backups:/app/backups
     environment:
       - TZ=UTC
       - SERVER_ADDRESS=:8080
-      - DATA_DIR=/app/data/gomft
-      - BACKUP_DIR=/app/data/gomft/backups
+      - DATA_DIR=/app/data
+      - BACKUP_DIR=/app/backups
       - JWT_SECRET=change_this_to_a_secure_random_string
       - BASE_URL=http://localhost:8080
       - EMAIL_ENABLED=true
@@ -143,6 +150,13 @@ services:
       - EMAIL_REQUIRE_AUTH=true
       - EMAIL_USERNAME=smtp_username
       - EMAIL_PASSWORD=smtp_password
+      # Logging configuration
+      - GOMFT_LOGS_DIR=/app/data/logs
+      - GOMFT_LOG_MAX_SIZE=10
+      - GOMFT_LOG_MAX_BACKUPS=5
+      - GOMFT_LOG_MAX_AGE=30
+      - GOMFT_LOG_COMPRESS=true
+      - GOMFT_LOG_LEVEL=info
 ```
 
 Alternatively, you can mount your own .env file to the container:
@@ -158,7 +172,7 @@ services:
       - "8080:8080"
     volumes:
       - ./data:/app/data
-      - ./backups:/app/data/gomft/backups
+      - ./backups:/app/backups
       - ./.env:/app/.env
     environment:
       - TZ=UTC
@@ -178,8 +192,8 @@ GoMFT uses an environment file located at `.env` in the root directory of the ap
 
 ```
 SERVER_ADDRESS=:8080
-DATA_DIR=./data/gomft
-BACKUP_DIR=./data/gomft/backups
+DATA_DIR=/app/data
+BACKUP_DIR=/app/backups
 JWT_SECRET=change_this_to_a_secure_random_string
 BASE_URL=http://localhost:8080
 
@@ -199,7 +213,7 @@ EMAIL_PASSWORD=smtp_password
 ### Configuration Options
 
 - `SERVER_ADDRESS`: The address and port to run the server on
-- `DATA_DIR`: Directory for storing application data
+- `DATA_DIR`: Directory for storing application data (database and configs)
 - `BACKUP_DIR`: Directory for storing database backups
 - `JWT_SECRET`: Secret key for JWT token generation
 - `BASE_URL`: Base URL for generating links in emails (e.g., password reset links)
@@ -215,6 +229,22 @@ EMAIL_PASSWORD=smtp_password
   - `EMAIL_ENABLE_TLS`: Set to `true` to use TLS for secure email transmission
   - `EMAIL_REQUIRE_AUTH`: Set to `true` to require authentication for SMTP connections, or `false` for servers that don't need authentication
 
+### Logging Configuration
+
+GoMFT provides configurable logging with rotation support through the following environment variables:
+
+- `GOMFT_LOGS_DIR`: Directory where log files are stored (default: `./data/logs`)
+- `GOMFT_LOG_MAX_SIZE`: Maximum size in megabytes for each log file before rotation (default: `10`)
+- `GOMFT_LOG_MAX_BACKUPS`: Number of old log files to retain (default: `5`)
+- `GOMFT_LOG_MAX_AGE`: Maximum number of days to retain old log files (default: `30`)
+- `GOMFT_LOG_COMPRESS`: Whether to compress rotated log files (default: `true`)
+- `GOMFT_LOG_LEVEL`: Controls verbosity level of logging (values: `error`, `info`, `debug`, default: `info`)
+  - `error`: Only show errors and critical issues
+  - `info`: Show errors and general operational information (default)
+  - `debug`: Show all messages including detailed debugging information
+
+Log files contain detailed information about file transfers, job execution, and system operations, which can be useful for troubleshooting and auditing.
+
 ## Usage
 
 1. Start the server:
@@ -233,6 +263,11 @@ EMAIL_PASSWORD=smtp_password
    - Navigate to "Transfer Configs" section
    - Configure source and destination locations with connection details
    - Set file patterns and archive options as needed
+   - Configure performance settings:
+     - Set "Concurrent Transfers" slider to optimize throughput
+     - Use higher values (8-16) for many small files or fast networks
+     - Use lower values (1-4) for large files or limited bandwidth
+     - Consider source/destination system capabilities when setting
 
 5. Create jobs using your configurations:
    - Navigate to "Jobs" section
@@ -294,8 +329,17 @@ User management features:
    - File patterns for filtering (e.g., `*.txt`, `data_*.csv`)
    - Output patterns for dynamic naming
    - Archive options for transferred files
+   - Skip already processed files to avoid duplicates
+   - Concurrent file transfers (configurable per job)
 
-4. **Schedule Options**:
+4. **Performance Options**:
+   - **Multi-threaded File Transfers**: Process multiple files simultaneously for higher throughput
+   - Configurable concurrency level (1-32 concurrent transfers)
+   - Per-job concurrency settings to optimize for different storage types
+   - Automatic transfer queue management to prevent overloading systems
+   - Adaptive processing based on source/destination capabilities
+
+5. **Schedule Options**:
    - Cron expressions for flexible scheduling
    - Manual execution
    - Enable/disable schedules
@@ -379,3 +423,23 @@ air
 ## License
 
 MIT License - see LICENSE file for details
+
+## Directory Structure
+
+GoMFT uses the following directory structure:
+
+- `/app/data`: Main application data directory
+  - Contains the SQLite database (`gomft.db`)
+  - Contains rclone configurations in `/app/data/configs`
+  - Contains log files in `/app/data/logs`
+- `/app/backups`: Database backup directory
+
+When using Docker, you should mount volumes to these locations:
+
+```yaml
+volumes:
+  - /host/path/data:/app/data         # For all application data
+  - /host/path/backups:/app/backups   # For database backups
+```
+
+These paths can be customized using the environment variables `DATA_DIR`, `BACKUP_DIR`, and `GOMFT_LOGS_DIR`.

components/config_form.templ

@@ -73,6 +73,7 @@ func getInitialData(config *db.TransferConfig) string {
 	archiveEnabled := false
 	deleteAfterTransfer := false
 	skipProcessedFiles := true
+	maxConcurrentTransfers := 4
 	rcloneFlags := ""
 
 	// If editing an existing config, populate with those values
@@ -131,6 +132,7 @@ func getInitialData(config *db.TransferConfig) string {
 		archiveEnabled = config.ArchiveEnabled
 		deleteAfterTransfer = config.DeleteAfterTransfer
 		skipProcessedFiles = config.SkipProcessedFiles
+		maxConcurrentTransfers = config.MaxConcurrentTransfers
 		rcloneFlags = config.RcloneFlags
 	}
 
@@ -186,6 +188,7 @@ func getInitialData(config *db.TransferConfig) string {
 		archiveEnabled: %v,
 		deleteAfterTransfer: %v,
 		skipProcessedFiles: %v,
+		maxConcurrentTransfers: %d,
 		rcloneFlags: '%s',
 	}`, 
 	name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
@@ -195,7 +198,7 @@ func getInitialData(config *db.TransferConfig) string {
 	destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
 	destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
 	destClientId, destClientSecret, destDriveId, destTeamDrive,
-	archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, rcloneFlags)
+	archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
 }
 
 templ ConfigForm(ctx context.Context, data ConfigFormData) {

components/providers/common/common.templ

@@ -141,6 +141,21 @@ templ ArchiveOptions() {
 				Files with the same hash that have been successfully processed before will be skipped
 			</p>
 		</div>
+
+		<div class="mb-4">
+			<label for="max_concurrent_transfers" class="block text-sm font-medium text-gray-700 mb-1">Concurrent Transfers</label>
+			<div class="flex items-center">
+				<input type="range" id="max_concurrent_transfers" name="max_concurrent_transfers" 
+					class="form-range w-2/3 h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer"
+					min="1" max="20" step="1"
+					x-model="maxConcurrentTransfers" 
+				/>
+				<span class="ml-4 text-gray-700" x-text="maxConcurrentTransfers"></span>
+			</div>
+			<p class="mt-1 text-xs text-gray-500">
+				Number of files to transfer simultaneously (higher values may improve performance but increase resource usage)
+			</p>
+		</div>
 	</div>
 }
 

docker-compose.yaml

@@ -8,14 +8,17 @@ services:
     ports:
       - "8080:8080"
     volumes:
-      # Persist data directory for SQLite database and configurations
+      # Main data directory - contains DB and configs
       - gomft-data:/app/data
+      # Separate backups directory
+      - gomft-backups:/app/backups
       # For development, you can mount the source code
       # - .:/app
     environment:
       - TZ=UTC
-      # Add any environment variables needed for configuration
-      # - GOMFT_DB_PATH=/app/data/gomft.db
+      - DATA_DIR=/app/data
+      - BACKUP_DIR=/app/backups
+      - GOMFT_LOGS_DIR=/app/data/logs
       # - GOMFT_LOG_LEVEL=info
     networks:
       - gomft-network
@@ -26,4 +29,6 @@ networks:
 
 volumes:
   gomft-data:
+    driver: local
+  gomft-backups:
     driver: local 

go.mod

@@ -44,6 +44,7 @@ require (
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.22.5 // indirect
 	modernc.org/mathutil v1.5.0 // indirect

go.sum

@@ -105,6 +105,8 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/config/config.go

@@ -2,7 +2,6 @@ package config
 
 import (
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -35,8 +34,8 @@ func Load() (*Config, error) {
 	// Default configuration
 	cfg := &Config{
 		ServerAddress: ":8080",
-		DataDir:       filepath.Join("./data", "gomft"),
-		BackupDir:     filepath.Join("./data", "gomft", "backups"),
+		DataDir:       "./data",
+		BackupDir:     "./backups",
 		JWTSecret:     "change_this_to_a_secure_random_string",
 		BaseURL:       "http://localhost:8080",
 		Email: EmailConfig{

internal/db/db.go

@@ -98,15 +98,16 @@ type TransferConfig struct {
 	DestDriveID      string `form:"dest_drive_id"`               // For OneDrive
 	DestTeamDrive    string `form:"dest_team_drive"`             // For Google Drive
 	// General fields
-	ArchivePath         string `form:"archive_path"`
-	ArchiveEnabled      bool   `gorm:"default:false" form:"archive_enabled"`
-	RcloneFlags         string `form:"rclone_flags"`
-	DeleteAfterTransfer bool   `gorm:"default:false" form:"delete_after_transfer"`
-	SkipProcessedFiles  bool   `gorm:"default:true" form:"skip_processed_files"`
-	CreatedBy           uint
-	User                User `gorm:"foreignkey:CreatedBy"`
-	CreatedAt           time.Time
-	UpdatedAt           time.Time
+	ArchivePath            string `form:"archive_path"`
+	ArchiveEnabled         bool   `gorm:"default:false" form:"archive_enabled"`
+	RcloneFlags            string `form:"rclone_flags"`
+	DeleteAfterTransfer    bool   `gorm:"default:false" form:"delete_after_transfer"`
+	SkipProcessedFiles     bool   `gorm:"default:true" form:"skip_processed_files"`
+	MaxConcurrentTransfers int    `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers
+	CreatedBy              uint
+	User                   User `gorm:"foreignkey:CreatedBy"`
+	CreatedAt              time.Time
+	UpdatedAt              time.Time
 }
 
 type Job struct {
@@ -381,7 +382,14 @@ func (db *DB) DeleteFileMetadata(id uint) error {
 
 // GetConfigRclonePath returns the path to the rclone config file for a given transfer config
 func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
-	return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
+	// Get data directory from environment or use default
+	dataDir := os.Getenv("DATA_DIR")
+	if dataDir == "" {
+		dataDir = "./data"
+	}
+
+	// Store configs in the data directory
+	return filepath.Join(dataDir, "configs", fmt.Sprintf("config_%d.conf", config.ID))
 }
 
 // GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
@@ -397,8 +405,11 @@ func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
 func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 	configPath := db.GetConfigRclonePath(config)
 
+	// Get the directory part of the path
+	configDir := filepath.Dir(configPath)
+
 	// Ensure configs directory exists
-	if err := os.MkdirAll("configs", 0755); err != nil {
+	if err := os.MkdirAll(configDir, 0755); err != nil {
 		return fmt.Errorf("failed to create configs directory: %v", err)
 	}
 
@@ -792,3 +803,9 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 
 	return nil
 }
+
+func (db *DB) GetActiveJobs() ([]Job, error) {
+	var jobs []Job
+	err := db.Preload("Config").Where("enabled = ?", true).Find(&jobs).Error
+	return jobs, err
+}

internal/db/migrations/add_max_concurrent_transfers.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddMaxConcurrentTransfersColumn adds the max_concurrent_transfers column to transfer_configs table
+func AddMaxConcurrentTransfersColumn() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "20250311_add_max_concurrent_transfers",
+		Migrate: func(tx *gorm.DB) error {
+			// Add max_concurrent_transfers column with default value of 4
+			return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN max_concurrent_transfers INTEGER DEFAULT 4").Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the column if needed
+			return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN max_concurrent_transfers").Error
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -12,6 +12,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		AddDeleteAfterTransferColumn(),
 		AddCloudStorageFields(),
 		AddSkipProcessedFilesColumn(),
+		AddMaxConcurrentTransfersColumn(),
 	}
 
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)