feat: Integrate rclone command functionality into configuration management

- Added support for selecting rclone commands and their associated flags in the configuration form.
- Implemented backend logic to handle rclone command retrieval and flag management.
- Enhanced the database schema to include rclone command and flag fields in the transfer configuration.
- Created new migration scripts to add rclone-related tables and fields.
- Updated the scheduler to execute transfers using the selected rclone command and flags.
- Developed new handlers for rendering rclone command options and flags in the web interface.
- Improved error handling and logging for rclone command execution and configuration updates.

components/config_form.templ

@@ -83,6 +83,8 @@ func getInitialData(config *db.TransferConfig) string {
 	skipProcessedFiles := true
 	maxConcurrentTransfers := 4
 	rcloneFlags := ""
+	commandId := uint(1) // Default to 'copy' command
+	commandFlags := ""
 	useBuiltinAuthSource := true
 	useBuiltinAuthDest := true
 
@@ -162,6 +164,8 @@ func getInitialData(config *db.TransferConfig) string {
 		skipProcessedFiles = config.GetSkipProcessedFiles()
 		maxConcurrentTransfers = config.MaxConcurrentTransfers
 		rcloneFlags = config.RcloneFlags
+		commandId = config.CommandID
+		commandFlags = config.CommandFlags
 		if config.UseBuiltinAuthSource != nil {
 			useBuiltinAuthSource = *config.UseBuiltinAuthSource
 		} else if sourceClientId != "" || sourceClientSecret != "" {
@@ -237,6 +241,8 @@ func getInitialData(config *db.TransferConfig) string {
 		skipProcessedFiles: %v,
 		maxConcurrentTransfers: %d,
 		rcloneFlags: '%s',
+		commandId: %d,
+		commandFlags: '%s',
 
 		// Path validation states
 		sourcePathValid: null,
@@ -274,7 +280,8 @@ func getInitialData(config *db.TransferConfig) string {
 	destClientId, destClientSecret, destDriveId, destTeamDrive,
 	destReadOnly, destStartYear, destIncludeArchived,
 	useBuiltinAuthSource, useBuiltinAuthDest,
-	archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
+	archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags, 
+	commandId, commandFlags)
 }
 
 templ ConfigForm(ctx context.Context, data ConfigFormData) {

components/providers/common/common.templ

@@ -1,5 +1,10 @@
 package common
 
+import (
+	"fmt"
+	"github.com/starfleetcptn/gomft/internal/db"
+)
+
 templ NameField() {
 <div class="mb-6">
 	<label for="name" class="block mb-2 text-sm font-medium text-gray-900 dark:text-white">Configuration Name</label>
@@ -135,7 +140,21 @@ templ ArchiveOptions() {
 
 templ RcloneFlags() {
 <div class="mb-6">
-	<label for="rclone_flags" class="block mb-2 text-sm font-medium text-gray-900 dark:text-white">Rclone Flags</label>
+	<label for="command_id" class="block mb-2 text-sm font-medium text-gray-900 dark:text-white">Rclone Command</label>
+	<div class="relative">
+		<div class="absolute inset-y-0 start-0 flex items-center ps-3.5 pointer-events-none">
+			<i class="fas fa-terminal text-gray-400 dark:text-gray-500"></i>
+		</div>
+		@RcloneCommandOptions()
+	</div>
+	<p class="mt-2 text-sm text-gray-500 dark:text-gray-400">
+		Select the rclone command to use for this configuration.
+	</p>
+
+	<!-- Command flags container - will be populated via HTMX -->
+	<div id="command-flags-container" class="mt-4"></div>
+
+	<label for="rclone_flags" class="block mb-2 mt-6 text-sm font-medium text-gray-900 dark:text-white">Additional Rclone Flags</label>
 	<div class="relative">
 		<div class="absolute inset-y-0 start-0 flex items-center ps-3.5 pointer-events-none">
 			<i class="fas fa-flag text-gray-400 dark:text-gray-500"></i>
@@ -150,6 +169,18 @@ templ RcloneFlags() {
 </div>
 }
 
+// New placeholder templ for rclone command options
+templ RcloneCommandOptions() {
+	<div 
+		hx-get="/api/rclone/commands" 
+		hx-trigger="load"
+		hx-target="this"
+		hx-swap="outerHTML">
+		<!-- Loading placeholder -->
+		<option value="">Loading commands...</option>
+	</div>
+}
+
 templ SourceSelection() {
 <div class="mb-6">
 	<label for="source_type" class="block mb-2 text-sm font-medium text-gray-900 dark:text-white">Source Type</label>
@@ -197,3 +228,79 @@ templ DestinationSelection() {
 	</div>
 </div>
 }
+
+// RcloneCommandOptionsContent renders the command options organized by category
+templ RcloneCommandOptionsContent(categoryMap map[string][]db.RcloneCommand, categories []string) {
+	<select id="command_id" name="command_id" x-model="commandId"
+		hx-get="/api/rclone/command-flags"
+		hx-target="#command-flags-container"
+		hx-trigger="change"
+		hx-include="[name='command_id']"
+		class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full ps-10 p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500">
+		<option value="">Select command...</option>
+		for _, category := range categories {
+			if commands, ok := categoryMap[category]; ok && len(commands) > 0 {
+				<optgroup label={ category }>
+					for _, cmd := range commands {
+						<option value={ fmt.Sprintf("%d", cmd.ID) }>{ cmd.Name } - { cmd.Description }</option>
+					}
+				</optgroup>
+			}
+		}
+	</select>
+}
+
+// RcloneCommandFlagsContent renders the command flags for a selected command
+templ RcloneCommandFlagsContent(command *db.RcloneCommand) {
+	if command == nil {
+		<div class="p-4 text-red-500">Command not found</div>
+		return
+	}
+
+	<div class="mb-4">
+		<h4 class="font-medium text-gray-900 dark:text-white mb-2">{ command.Name }</h4>
+		<p class="text-sm text-gray-600 dark:text-gray-400 mb-4">{ command.Description }</p>
+		
+		if len(command.Flags) > 0 {
+			<div class="mb-3">
+				<h5 class="font-medium text-gray-900 dark:text-white mb-2">Command Flags:</h5>
+				<div class="space-y-3">
+					for _, flag := range command.Flags {
+						<div class="flex items-start">
+							<input
+								type="checkbox"
+								id={ fmt.Sprintf("flag_%d", flag.ID) }
+								name="command_flags"
+								value={ fmt.Sprintf("%d", flag.ID) }
+								class="mt-0.5 rounded border-gray-300 text-blue-600 focus:ring-blue-500 dark:border-gray-600 dark:bg-gray-700 dark:focus:ring-blue-600"
+							/>
+							<div class="ml-3">
+								<label for={ fmt.Sprintf("flag_%d", flag.ID) } class="font-medium text-gray-900 dark:text-white">
+									{ flag.Name } - { flag.Description }
+								</label>
+								if flag.DefaultValue != "" {
+									<p class="text-xs text-gray-500 dark:text-gray-400">Default: { flag.DefaultValue }</p>
+								}
+							</div>
+						</div>
+					}
+				</div>
+			</div>
+			
+			<div class="mt-4 p-3 bg-blue-50 text-blue-800 rounded-lg border border-blue-100 dark:bg-blue-900/20 dark:text-blue-300 dark:border-blue-900 text-sm">
+				<div class="flex items-center mb-1">
+					<i class="fas fa-info-circle mr-2"></i>
+					<span class="font-medium">Usage Example</span>
+				</div>
+				<code class="block mt-1 font-mono text-xs overflow-x-auto">rclone { command.Name } [flags] source:path dest:path</code>
+			</div>
+		} else {
+			<div class="p-3 bg-gray-50 text-gray-600 rounded-lg border border-gray-200 dark:bg-gray-800 dark:text-gray-400 dark:border-gray-700 text-sm">
+				<div class="flex items-center">
+					<i class="fas fa-info-circle mr-2"></i>
+					<span>This command doesn't have any specific flags.</span>
+				</div>
+			</div>
+		}
+	</div>
+}

internal/db/db.go

@@ -123,6 +123,9 @@ type TransferConfig struct {
 	ArchivePath    string `form:"archive_path"`
 	ArchiveEnabled *bool  `gorm:"default:false" form:"archive_enabled"`
 	RcloneFlags    string `form:"rclone_flags"`
+	// Rclone command fields
+	CommandID              uint   `gorm:"default:1" form:"command_id"` // Default to 'copy' command ID (1)
+	CommandFlags           string `form:"command_flags"`               // JSON string of selected flags
 	DeleteAfterTransfer    *bool  `gorm:"default:false" form:"delete_after_transfer"`
 	SkipProcessedFiles     *bool  `gorm:"default:true" form:"skip_processed_files"`
 	MaxConcurrentTransfers int    `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers
@@ -250,6 +253,31 @@ type DB struct {
 	*gorm.DB
 }
 
+// RcloneCommand represents a command available in rclone
+type RcloneCommand struct {
+	ID          uint                `gorm:"primarykey"`
+	Name        string              `gorm:"not null;uniqueIndex"`
+	Description string              `gorm:"not null"`
+	Category    string              `gorm:"not null;index"`
+	IsAdvanced  bool                `gorm:"not null;default:false"`
+	Flags       []RcloneCommandFlag `gorm:"foreignKey:CommandID;constraint:OnDelete:CASCADE"`
+	CreatedAt   time.Time           `gorm:"not null"`
+}
+
+// RcloneCommandFlag represents a flag that can be used with an rclone command
+type RcloneCommandFlag struct {
+	ID           uint          `gorm:"primarykey"`
+	CommandID    uint          `gorm:"not null;index"`
+	Command      RcloneCommand `gorm:"foreignKey:CommandID"`
+	Name         string        `gorm:"not null;index"`
+	ShortName    string
+	Description  string `gorm:"not null"`
+	DataType     string `gorm:"not null"` // string, int, bool, etc.
+	IsRequired   bool   `gorm:"not null;default:false"`
+	DefaultValue string
+	CreatedAt    time.Time `gorm:"not null"`
+}
+
 func Initialize(dbPath string) (*DB, error) {
 	// Create directory if it doesn't exist
 	dir := filepath.Dir(dbPath)
@@ -1707,3 +1735,346 @@ func (u *User) CheckPassword(password string) bool {
 	err := bcrypt.CompareHashAndPassword([]byte(u.PasswordHash), []byte(password))
 	return err == nil
 }
+
+// GetRcloneCommands returns all rclone commands
+func (db *DB) GetRcloneCommands() ([]RcloneCommand, error) {
+	var commands []RcloneCommand
+	err := db.Find(&commands).Error
+	return commands, err
+}
+
+// GetRcloneCommand returns a specific rclone command by ID
+func (db *DB) GetRcloneCommand(id uint) (*RcloneCommand, error) {
+	var command RcloneCommand
+	err := db.First(&command, id).Error
+	if err != nil {
+		return nil, err
+	}
+	return &command, nil
+}
+
+// GetRcloneCommandByName returns a specific rclone command by name
+func (db *DB) GetRcloneCommandByName(name string) (*RcloneCommand, error) {
+	var command RcloneCommand
+	err := db.Where("name = ?", name).First(&command).Error
+	if err != nil {
+		return nil, err
+	}
+	return &command, nil
+}
+
+// GetRcloneCommandsInCategory returns all commands in a specific category
+func (db *DB) GetRcloneCommandsInCategory(category string) ([]RcloneCommand, error) {
+	var commands []RcloneCommand
+	err := db.Where("category = ?", category).Find(&commands).Error
+	return commands, err
+}
+
+// GetRcloneCommandFlag returns a specific flag by ID
+func (db *DB) GetRcloneCommandFlag(id uint) (*RcloneCommandFlag, error) {
+	var flag RcloneCommandFlag
+	err := db.First(&flag, id).Error
+	if err != nil {
+		return nil, err
+	}
+	return &flag, nil
+}
+
+// GetRcloneCommandFlagByName returns a specific flag by name for a command
+func (db *DB) GetRcloneCommandFlagByName(commandID uint, name string) (*RcloneCommandFlag, error) {
+	var flag RcloneCommandFlag
+	err := db.Where("command_id = ? AND name = ?", commandID, name).First(&flag).Error
+	if err != nil {
+		return nil, err
+	}
+	return &flag, nil
+}
+
+// GetRcloneCommandFlags returns all flags for a specific command
+func (db *DB) GetRcloneCommandFlags(commandID uint) ([]RcloneCommandFlag, error) {
+	var flags []RcloneCommandFlag
+	err := db.Where("command_id = ?", commandID).Find(&flags).Error
+	return flags, err
+}
+
+// GetRcloneCommandWithFlags returns a command with all its flags
+func (db *DB) GetRcloneCommandWithFlags(commandID uint) (*RcloneCommand, error) {
+	var command RcloneCommand
+	err := db.Preload("Flags").First(&command, commandID).Error
+	if err != nil {
+		return nil, err
+	}
+	return &command, nil
+}
+
+// BuildRcloneCommand builds an rclone command string with the specified command and flags
+func (db *DB) BuildRcloneCommand(commandName string, flags map[string]string) (string, error) {
+	// Get the command details
+	command, err := db.GetRcloneCommandByName(commandName)
+	if err != nil {
+		return "", fmt.Errorf("command not found: %s", commandName)
+	}
+
+	// Start building the command string
+	cmdStr := "rclone " + command.Name
+
+	// Get all flags for this command
+	allFlags, err := db.GetRcloneCommandFlags(command.ID)
+	if err != nil {
+		return "", fmt.Errorf("failed to get flags for command: %v", err)
+	}
+
+	// Create a map of flag details for easy lookup
+	flagDetails := make(map[string]RcloneCommandFlag)
+	for _, f := range allFlags {
+		flagDetails[f.Name] = f
+	}
+
+	// Add the flags to the command
+	for name, value := range flags {
+		// Check if the flag exists for this command
+		flag, exists := flagDetails[name]
+		if !exists {
+			return "", fmt.Errorf("invalid flag for command %s: %s", commandName, name)
+		}
+
+		// Handle different flag types
+		switch flag.DataType {
+		case "bool":
+			if value == "true" {
+				cmdStr += " " + name
+			}
+		default:
+			cmdStr += " " + name + " " + value
+		}
+	}
+
+	return cmdStr, nil
+}
+
+// ValidateRcloneFlags validates if the provided flags are valid for the command
+func (db *DB) ValidateRcloneFlags(commandName string, flags map[string]string) (bool, map[string]string) {
+	// Initialize errors map
+	errors := make(map[string]string)
+
+	// Get the command details
+	command, err := db.GetRcloneCommandByName(commandName)
+	if err != nil {
+		errors["command"] = "Command not found: " + commandName
+		return false, errors
+	}
+
+	// Get all flags for this command
+	allFlags, err := db.GetRcloneCommandFlags(command.ID)
+	if err != nil {
+		errors["command"] = "Failed to get flags for command"
+		return false, errors
+	}
+
+	// Create a map of flag details for easy lookup
+	flagDetails := make(map[string]RcloneCommandFlag)
+	for _, f := range allFlags {
+		flagDetails[f.Name] = f
+	}
+
+	// Check each provided flag
+	for name, value := range flags {
+		// Check if the flag exists for this command
+		flag, exists := flagDetails[name]
+		if !exists {
+			errors[name] = "Invalid flag for command " + commandName
+			continue
+		}
+
+		// Validate the flag value based on data type
+		switch flag.DataType {
+		case "int":
+			if _, err := strconv.Atoi(value); err != nil {
+				errors[name] = "Value must be an integer"
+			}
+		case "float":
+			if _, err := strconv.ParseFloat(value, 64); err != nil {
+				errors[name] = "Value must be a number"
+			}
+		case "bool":
+			if value != "true" && value != "false" {
+				errors[name] = "Value must be true or false"
+			}
+		}
+	}
+
+	// Check for required flags
+	for _, flag := range allFlags {
+		if flag.IsRequired {
+			if _, provided := flags[flag.Name]; !provided {
+				errors[flag.Name] = "This flag is required"
+			}
+		}
+	}
+
+	return len(errors) == 0, errors
+}
+
+// GetRcloneCategories returns all unique categories of rclone commands
+func (db *DB) GetRcloneCategories() ([]string, error) {
+	var categories []string
+	err := db.Model(&RcloneCommand{}).Distinct("category").Pluck("category", &categories).Error
+	return categories, err
+}
+
+// GetRcloneCommandsByAdvanced returns commands filtered by their advanced status
+func (db *DB) GetRcloneCommandsByAdvanced(isAdvanced bool) ([]RcloneCommand, error) {
+	var commands []RcloneCommand
+	err := db.Where("is_advanced = ?", isAdvanced).Find(&commands).Error
+	return commands, err
+}
+
+// SearchRcloneCommands searches for commands by name or description
+func (db *DB) SearchRcloneCommands(query string) ([]RcloneCommand, error) {
+	var commands []RcloneCommand
+	searchQuery := "%" + query + "%"
+	err := db.Where("name LIKE ? OR description LIKE ?", searchQuery, searchQuery).Find(&commands).Error
+	return commands, err
+}
+
+// GetRcloneFlagUsage returns a human-readable usage example for a flag
+func (flag *RcloneCommandFlag) GetUsageExample() string {
+	switch flag.DataType {
+	case "bool":
+		return flag.Name
+	case "int":
+		return fmt.Sprintf("%s=<number>", flag.Name)
+	case "float":
+		return fmt.Sprintf("%s=<decimal>", flag.Name)
+	case "string":
+		return fmt.Sprintf("%s=<text>", flag.Name)
+	default:
+		return fmt.Sprintf("%s=<value>", flag.Name)
+	}
+}
+
+// GetRcloneCommandUsage returns a basic usage example for a command with its required flags
+func (db *DB) GetRcloneCommandUsage(commandID uint) (string, error) {
+	command, err := db.GetRcloneCommandWithFlags(commandID)
+	if err != nil {
+		return "", err
+	}
+
+	usage := fmt.Sprintf("rclone %s [flags] <source> <dest>", command.Name)
+
+	// Add basic usage examples for required flags
+	requiredFlags := []string{}
+	for _, flag := range command.Flags {
+		if flag.IsRequired {
+			requiredFlags = append(requiredFlags, flag.GetUsageExample())
+		}
+	}
+
+	if len(requiredFlags) > 0 {
+		usage += "\n\nRequired flags:\n  " + strings.Join(requiredFlags, "\n  ")
+	}
+
+	return usage, nil
+}
+
+// ParseRcloneFlags parses a string of rclone flags into a map
+func ParseRcloneFlags(flagsStr string) map[string]string {
+	result := make(map[string]string)
+	if flagsStr == "" {
+		return result
+	}
+
+	// Split the flags string by spaces
+	parts := strings.Fields(flagsStr)
+
+	for i := 0; i < len(parts); i++ {
+		part := parts[i]
+
+		// Check if it's a flag (starts with --)
+		if strings.HasPrefix(part, "--") {
+			// Remove the -- prefix
+			flagName := part
+
+			// Check if the flag has a value
+			if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "--") {
+				// Next part is a value
+				result[flagName] = parts[i+1]
+				i++ // Skip the value in the next iteration
+			} else {
+				// Flag without value, treat as boolean
+				result[flagName] = "true"
+			}
+		}
+	}
+
+	return result
+}
+
+// RenderRcloneCommandHelp generates a help text for a command with its flags
+func (db *DB) RenderRcloneCommandHelp(commandID uint) (string, error) {
+	command, err := db.GetRcloneCommandWithFlags(commandID)
+	if err != nil {
+		return "", err
+	}
+
+	// Build the help text
+	help := fmt.Sprintf("COMMAND: %s\n", command.Name)
+	help += fmt.Sprintf("DESCRIPTION: %s\n\n", command.Description)
+	help += "FLAGS:\n"
+
+	// Group flags by required status
+	var requiredFlags, optionalFlags []RcloneCommandFlag
+	for _, flag := range command.Flags {
+		if flag.IsRequired {
+			requiredFlags = append(requiredFlags, flag)
+		} else {
+			optionalFlags = append(optionalFlags, flag)
+		}
+	}
+
+	// Add required flags
+	if len(requiredFlags) > 0 {
+		help += "  Required:\n"
+		for _, flag := range requiredFlags {
+			shortName := ""
+			if flag.ShortName != "" {
+				shortName = fmt.Sprintf(" (-%s)", flag.ShortName)
+			}
+			help += fmt.Sprintf("    %s%s - %s\n", flag.Name, shortName, flag.Description)
+			if flag.DataType != "bool" && flag.DefaultValue != "" {
+				help += fmt.Sprintf("      Default: %s\n", flag.DefaultValue)
+			}
+		}
+	}
+
+	// Add optional flags
+	if len(optionalFlags) > 0 {
+		help += "\n  Optional:\n"
+		for _, flag := range optionalFlags {
+			shortName := ""
+			if flag.ShortName != "" {
+				shortName = fmt.Sprintf(" (-%s)", flag.ShortName)
+			}
+			help += fmt.Sprintf("    %s%s - %s\n", flag.Name, shortName, flag.Description)
+			if flag.DataType != "bool" && flag.DefaultValue != "" {
+				help += fmt.Sprintf("      Default: %s\n", flag.DefaultValue)
+			}
+		}
+	}
+
+	return help, nil
+}
+
+func (db *DB) GetRcloneCommandFlagsMap(commandID uint) (map[uint]RcloneCommandFlag, error) {
+	flags, err := db.GetRcloneCommandFlags(commandID)
+	if err != nil {
+		return nil, err
+	}
+
+	flagsMap := make(map[uint]RcloneCommandFlag)
+	for _, flag := range flags {
+		flagsMap[flag.ID] = flag
+	}
+
+	return flagsMap, nil
+}

internal/db/migrations/009_add_rclone_tables.go

@@ -0,0 +1,349 @@
+package migrations
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddRcloneTables adds tables for rclone commands and their flags
+func AddRcloneTables() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "009_add_rclone_tables",
+		Migrate: func(tx *gorm.DB) error {
+			// Check if any tables exist (indicating an existing database)
+			var count int64
+			if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil {
+				return fmt.Errorf("failed to check for existing tables: %v", err)
+			}
+
+			// If tables exist, create a backup
+			if count > 0 {
+				// Get the database path
+				sqlDB, err := tx.DB()
+				if err != nil {
+					return fmt.Errorf("failed to get underlying database: %v", err)
+				}
+
+				var seq int
+				var name, dbPath string
+				if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil {
+					return fmt.Errorf("failed to get database path: %v", err)
+				}
+
+				// Get backup directory from environment variable or use default
+				backupDir := os.Getenv("BACKUP_DIR")
+				if backupDir == "" {
+					backupDir = "/app/backups" // Default Docker path
+					// Check if we're not in Docker
+					if _, err := os.Stat(backupDir); os.IsNotExist(err) {
+						backupDir = "backups" // Fallback to local directory
+					}
+				}
+
+				// Create backup directory if it doesn't exist
+				if err := os.MkdirAll(backupDir, 0755); err != nil {
+					return fmt.Errorf("failed to create backup directory: %v", err)
+				}
+
+				// Create backup file with timestamp in the backup directory
+				dbFileName := filepath.Base(dbPath)
+				backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405"))
+				backupFile := filepath.Join(backupDir, backupFileName)
+
+				// Read original database
+				data, err := os.ReadFile(dbPath)
+				if err != nil {
+					return fmt.Errorf("failed to read database for backup: %v", err)
+				}
+
+				// Write backup
+				if err := os.WriteFile(backupFile, data, 0600); err != nil {
+					return fmt.Errorf("failed to create database backup: %v", err)
+				}
+
+				fmt.Printf("Created database backup at: %s\n", backupFile)
+			}
+
+			// Create the rclone_commands table
+			err := tx.Exec(`
+				CREATE TABLE IF NOT EXISTS rclone_commands (
+					id INTEGER PRIMARY KEY AUTOINCREMENT,
+					name TEXT NOT NULL,
+					description TEXT NOT NULL,
+					category TEXT NOT NULL,
+					is_advanced BOOLEAN NOT NULL DEFAULT 0,
+					created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
+				)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Create an index on name for faster lookups
+			err = tx.Exec(`
+				CREATE UNIQUE INDEX IF NOT EXISTS idx_rclone_commands_name ON rclone_commands(name)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Create an index on category for faster filtering
+			err = tx.Exec(`
+				CREATE INDEX IF NOT EXISTS idx_rclone_commands_category ON rclone_commands(category)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Create the rclone_command_flags table
+			err = tx.Exec(`
+				CREATE TABLE IF NOT EXISTS rclone_command_flags (
+					id INTEGER PRIMARY KEY AUTOINCREMENT,
+					command_id INTEGER NOT NULL,
+					name TEXT NOT NULL,
+					short_name TEXT,
+					description TEXT NOT NULL,
+					data_type TEXT NOT NULL,
+					is_required BOOLEAN NOT NULL DEFAULT 0,
+					default_value TEXT,
+					created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+					FOREIGN KEY (command_id) REFERENCES rclone_commands(id) ON DELETE CASCADE
+				)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Create index on command_id for faster lookups
+			err = tx.Exec(`
+				CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_command_id ON rclone_command_flags(command_id)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Create index on flag name for faster searches
+			err = tx.Exec(`
+				CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_name ON rclone_command_flags(name)
+			`).Error
+			if err != nil {
+				return err
+			}
+
+			// Populate the tables with default rclone commands and flags
+			return populateRcloneTablesWithDefaults(tx)
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the flags table first due to foreign key constraints
+			err := tx.Exec(`DROP TABLE IF EXISTS rclone_command_flags`).Error
+			if err != nil {
+				return err
+			}
+
+			// Then drop the commands table
+			return tx.Exec(`DROP TABLE IF EXISTS rclone_commands`).Error
+		},
+	}
+}
+
+// Helper function to populate rclone tables with default data
+func populateRcloneTablesWithDefaults(tx *gorm.DB) error {
+	// Define base commands
+	commands := []struct {
+		Name        string
+		Description string
+		Category    string
+		IsAdvanced  bool
+	}{
+		// Core commands
+		{"copy", "Copy files from source to dest, skipping already copied", "sync", false},
+		{"sync", "Make source and dest identical, modifying destination only", "sync", false},
+		{"bisync", "Bidirectional synchronization between two paths", "sync", false},
+		{"move", "Move files from source to dest", "sync", false},
+		{"delete", "Remove the contents of path", "sync", false},
+		{"purge", "Remove the path and all of its contents", "sync", false},
+		{"mkdir", "Make the path if it doesn't already exist", "sync", false},
+		{"rmdir", "Remove the path", "sync", false},
+		{"rmdirs", "Remove any empty directories under the path", "sync", false},
+		{"check", "Check if the files in the source and destination match", "sync", false},
+		{"ls", "List all the objects in the path with size and path", "listing", false},
+		{"lsd", "List all directories/containers/buckets in the path", "listing", false},
+		{"lsl", "List all the objects in the path with size, modification time and path", "listing", false},
+		{"lsf", "List the objects in the path (obey formatting parameters)", "listing", false},
+		{"lsjson", "List the objects in the path in JSON format", "listing", false},
+		{"md5sum", "Produce an md5sum file for all the objects in the path", "hash", false},
+		{"sha1sum", "Produce a sha1sum file for all the objects in the path", "hash", false},
+		{"size", "Return the total size and number of objects in remote:path", "info", false},
+		{"version", "Show the version number", "info", false},
+		{"cleanup", "Clean up the remote if possible", "maintenance", false},
+		{"dedupe", "Interactively find duplicate files and delete/rename them", "maintenance", false},
+		{"copyto", "Copy files from source to dest, skipping already copied", "sync", false},
+		{"moveto", "Move file or directory from source to dest", "sync", false},
+		{"listremotes", "List all the remotes in the config file", "config", false},
+		{"obscure", "Obscure password for use in the rclone.conf", "config", false},
+		{"cryptcheck", "Check the integrity of an encrypted remote", "crypto", false},
+	}
+
+	// Insert commands
+	for _, cmd := range commands {
+		err := tx.Exec(`
+			INSERT INTO rclone_commands 
+			(name, description, category, is_advanced, created_at) 
+			VALUES (?, ?, ?, ?, ?)
+		`, cmd.Name, cmd.Description, cmd.Category, cmd.IsAdvanced, time.Now()).Error
+
+		if err != nil {
+			return fmt.Errorf("failed to insert command %s: %v", cmd.Name, err)
+		}
+	}
+
+	// Define all flags
+	allFlags := []struct {
+		CommandName  string
+		Name         string
+		ShortName    string
+		Description  string
+		DataType     string
+		IsRequired   bool
+		DefaultValue string
+	}{
+		// Global flags - apply to most commands
+		{"global", "transfers", "n", "Number of file transfers to run in parallel", "int", false, "4"},
+		{"global", "checkers", "", "Number of checkers to run in parallel", "int", false, "8"},
+		{"global", "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "string", false, "NOTICE"},
+		{"global", "stats", "", "Interval between logging stats, e.g 500ms, 60s, 5m", "string", false, "1m0s"},
+		{"global", "stats-file-name-length", "", "Max file name length in stats", "int", false, "45"},
+		{"global", "stats-one-line", "", "Make the stats fit on one line", "bool", false, "false"},
+		{"global", "progress", "p", "Show progress during transfer", "bool", false, "false"},
+		{"global", "verbose", "v", "Show verbose output", "bool", false, "false"},
+		{"global", "quiet", "q", "Print as little stuff as possible", "bool", false, "false"},
+		{"global", "retries", "", "Retry operations this many times if they fail", "int", false, "3"},
+		{"global", "retries-sleep", "", "Interval between retrying operations if they fail", "string", false, "0s"},
+		{"global", "timeout", "", "IO idle timeout", "string", false, "5m0s"},
+		{"global", "tpslimit", "", "Limit HTTP transactions per second", "float", false, "0"},
+		{"global", "tpslimit-burst", "", "Max burst of transactions for --tpslimit", "int", false, "1"},
+		{"global", "size-only", "", "Skip based on size only, not mod-time or checksum", "bool", false, "false"},
+		{"global", "ignore-checksum", "", "Skip post copy check of checksums", "bool", false, "false"},
+		{"global", "ignore-existing", "", "Skip all files that exist on destination", "bool", false, "false"},
+		{"global", "ignore-size", "", "Skip size checks to calculate if file changed", "bool", false, "false"},
+		{"global", "ignore-case-sync", "", "Ignore case when synchronizing", "bool", false, "false"},
+		{"global", "no-update-modtime", "", "Don't update destination mod-time if files identical", "bool", false, "false"},
+		{"global", "no-check-certificate", "", "Do not verify server SSL certificates", "bool", false, "false"},
+		{"global", "ask-password", "", "Allow prompt for password for encrypted config", "bool", false, "false"},
+		{"global", "dump", "", "List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles", "string", false, ""},
+		{"global", "metadata", "M", "Preserve metadata when copying objects", "bool", false, "false"},
+		{"global", "metadata-set", "", "Add metadata key=value when uploading", "string", false, ""},
+
+		// copy command specific flags
+		{"copy", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+		{"copy", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
+		{"copy", "cutoff-mode", "", "Mode to stop transfers when reaching the cutoff threshold", "string", false, "hard"},
+		{"copy", "max-transfer", "", "Maximum size of data to transfer", "string", false, "off"},
+		{"copy", "max-backlog", "", "Maximum size of upload or download backlog", "int", false, "10000"},
+		{"copy", "track-renames", "", "Track file renames during sync", "bool", false, "false"},
+		{"copy", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"},
+
+		// sync command specific flags
+		{"sync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+		{"sync", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
+		{"sync", "backup-dir", "", "Make backups into this directory", "string", false, ""},
+		{"sync", "suffix", "", "Suffix to add to changed files", "string", false, ""},
+		{"sync", "delete-before", "", "Delete before transferring", "bool", false, "false"},
+		{"sync", "delete-during", "", "Delete during transferring", "bool", false, "false"},
+		{"sync", "delete-after", "", "Delete after transferring", "bool", false, "false"},
+		{"sync", "track-renames", "", "Track file renames during sync", "bool", false, "false"},
+		{"sync", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"},
+
+		// bisync command specific flags
+		{"bisync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+		{"bisync", "resync", "", "Performs the resync run", "bool", false, "false"},
+		{"bisync", "check-access", "", "Ensure destination has write access", "bool", false, "true"},
+		{"bisync", "conflict-resolve", "", "Automatically resolve conflicts (newer|larger|older|smaller)", "string", false, ""},
+		{"bisync", "max-delete", "", "Safety check on maximum files to delete", "int", false, "10"},
+
+		// move command specific flags
+		{"move", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+		{"move", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
+		{"move", "delete-empty-src-dirs", "", "Delete empty source dirs after move", "bool", false, "false"},
+
+		// ls/listing related flags
+		{"ls", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
+		{"ls", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
+		{"ls", "format", "", "Format for the output", "string", false, ""},
+		{"ls", "absolute", "", "Put a leading / in front of path names", "bool", false, "false"},
+
+		{"lsl", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
+		{"lsl", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
+
+		{"lsd", "max-depth", "", "Maximum depth to show in the listing", "int", false, ""},
+		{"lsd", "dir-sort", "", "Directory sorting (alphabetical|size|time)", "string", false, "alphabetical"},
+
+		{"lsf", "format", "", "Format for the output", "string", false, ""},
+		{"lsf", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
+		{"lsf", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
+
+		{"lsjson", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
+		{"lsjson", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
+		{"lsjson", "files-only", "", "Show only files, not directories", "bool", false, "false"},
+		{"lsjson", "encrypted", "", "Show the encrypted names", "bool", false, "false"},
+		{"lsjson", "stat", "", "Show status of objects", "bool", false, "false"},
+
+		// Other specialized command flags
+		{"cryptcheck", "one-way", "", "Check one way only, source files must exist on destination", "bool", false, "false"},
+
+		{"cleanup", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+
+		{"dedupe", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
+		{"dedupe", "mode", "", "Dedupe mode interactive|skip|first|newest|oldest|largest|smallest|rename", "string", false, "interactive"},
+	}
+
+	// Insert flags with associated command IDs
+	for _, flag := range allFlags {
+		// For global flags, add them to all commands
+		if flag.CommandName == "global" {
+			// Get all command IDs
+			var commandIDs []int64
+			err := tx.Raw(`SELECT id FROM rclone_commands`).Scan(&commandIDs).Error
+			if err != nil {
+				return fmt.Errorf("failed to get all command IDs: %v", err)
+			}
+
+			// Add global flag to each command
+			for _, cmdID := range commandIDs {
+				err = tx.Exec(`
+					INSERT INTO rclone_command_flags 
+					(command_id, name, short_name, description, data_type, is_required, default_value, created_at) 
+					VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+				`, cmdID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error
+
+				if err != nil {
+					return fmt.Errorf("failed to insert global flag %s for command ID %d: %v", flag.Name, cmdID, err)
+				}
+			}
+		} else {
+			// Add command-specific flag
+			var commandID int64
+			err := tx.Raw(`SELECT id FROM rclone_commands WHERE name = ?`, flag.CommandName).Scan(&commandID).Error
+			if err != nil {
+				return fmt.Errorf("failed to find command %s: %v", flag.CommandName, err)
+			}
+
+			err = tx.Exec(`
+				INSERT INTO rclone_command_flags 
+				(command_id, name, short_name, description, data_type, is_required, default_value, created_at) 
+				VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+			`, commandID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error
+
+			if err != nil {
+				return fmt.Errorf("failed to insert flag %s for command %s: %v", flag.Name, flag.CommandName, err)
+			}
+		}
+	}
+
+	return nil
+}

internal/db/migrations/010_add_rclone_command_to_config.go

@@ -0,0 +1,96 @@
+package migrations
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddRcloneCommandToConfig adds rclone command fields to the transfer_configs table
+func AddRcloneCommandToConfig() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "010_add_rclone_command_to_config",
+		Migrate: func(tx *gorm.DB) error {
+			// Check if any tables exist (indicating an existing database)
+			var count int64
+			if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil {
+				return fmt.Errorf("failed to check for existing tables: %v", err)
+			}
+
+			// If tables exist, create a backup
+			if count > 0 {
+				// Get the database path
+				sqlDB, err := tx.DB()
+				if err != nil {
+					return fmt.Errorf("failed to get underlying database: %v", err)
+				}
+
+				var seq int
+				var name, dbPath string
+				if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil {
+					return fmt.Errorf("failed to get database path: %v", err)
+				}
+
+				// Get backup directory from environment variable or use default
+				backupDir := os.Getenv("BACKUP_DIR")
+				if backupDir == "" {
+					backupDir = "/app/backups" // Default Docker path
+					// Check if we're not in Docker
+					if _, err := os.Stat(backupDir); os.IsNotExist(err) {
+						backupDir = "backups" // Fallback to local directory
+					}
+				}
+
+				// Create backup directory if it doesn't exist
+				if err := os.MkdirAll(backupDir, 0755); err != nil {
+					return fmt.Errorf("failed to create backup directory: %v", err)
+				}
+
+				// Create backup file with timestamp in the backup directory
+				dbFileName := filepath.Base(dbPath)
+				backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405"))
+				backupFile := filepath.Join(backupDir, backupFileName)
+
+				// Read original database
+				data, err := os.ReadFile(dbPath)
+				if err != nil {
+					return fmt.Errorf("failed to read database for backup: %v", err)
+				}
+
+				// Write backup
+				if err := os.WriteFile(backupFile, data, 0600); err != nil {
+					return fmt.Errorf("failed to create database backup: %v", err)
+				}
+
+				fmt.Printf("Created database backup at: %s\n", backupFile)
+			}
+
+			// Add the command_id and command_flags columns to the transfer_configs table
+			if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_id INTEGER DEFAULT NULL REFERENCES rclone_commands(id)`).Error; err != nil {
+				return err
+			}
+
+			if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_flags TEXT DEFAULT NULL`).Error; err != nil {
+				return err
+			}
+
+			return nil
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Remove the columns in reverse order
+			if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_flags`).Error; err != nil {
+				return err
+			}
+
+			if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_id`).Error; err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -19,6 +19,8 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		AddTimestampsToJobHistories(), // 006
 		AddNotificationServices(),     // 007
 		AddUserNotifications(),        // 008
+		AddRcloneTables(),             // 009
+		AddRcloneCommandToConfig(),    // 010
 	)
 
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)

internal/scheduler/scheduler.go

@@ -464,6 +464,30 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig,
 	// Get rclone config path
 	configPath := s.db.GetConfigRclonePath(&config)
 
+	// Get the command to use for the transfer
+	var rcloneCommand string = "copyto" // Default command
+	if config.CommandID > 0 {
+		// Get the command by ID
+		command, err := s.db.GetRcloneCommand(config.CommandID)
+		if err == nil && command != nil {
+			rcloneCommand = command.Name
+			s.log.LogDebug("Using rclone command %s for job %d, config %d", rcloneCommand, job.ID, config.ID)
+		} else {
+			s.log.LogError("Failed to get rclone command with ID %d: %v", config.CommandID, err)
+		}
+	}
+
+	// Determine command type to handle execution appropriately
+	commandType := determineCommandType(rcloneCommand)
+	s.log.LogDebug("Command %s is of type: %s", rcloneCommand, commandType)
+
+	// For non-file-by-file transfer commands, use the simple execution approach
+	if commandType != "transfer" || isDirectoryBasedTransfer(rcloneCommand) {
+		s.executeSimpleCommand(rcloneCommand, commandType, job, config, history, configPath)
+		return
+	}
+
+	// The rest of the function handles file-by-file transfer commands (copyto, moveto)
 	// Use lsjson to get file list and metadata in one operation instead of separate size and ls commands
 	listArgs := []string{
 		"--config", configPath,
@@ -564,7 +588,7 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig,
 	var files []map[string]interface{}
 	var totalSize int64
 	for _, entry := range fileEntries {
-		// Skip directories
+		// Process directories
 		if isDir, ok := entry["IsDir"].(bool); ok && isDir {
 			continue
 		}
@@ -753,16 +777,43 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig,
 				wg.Done()
 			}()
 
-			// Prepare moveto command for transfer
+			// Prepare rclone command
 			transferArgs := []string{
 				"--config", configPath,
-				"copyto",
 				"--progress",
 				"--stats-one-line",
 				"--verbose",
 				"--stats", "1s",
 			}
 
+			// Add the command to the arguments
+			transferArgs = append(transferArgs, rcloneCommand)
+
+			// Add command flags if specified
+			if config.CommandFlags != "" {
+				var flagIDs []uint
+				if err := json.Unmarshal([]byte(config.CommandFlags), &flagIDs); err == nil {
+					// Get the flags for the selected command
+					for _, flagID := range flagIDs {
+						flag, err := s.db.GetRcloneCommandFlag(flagID)
+						if err == nil && flag != nil {
+							if flag.DataType == "bool" {
+								// For boolean flags, just add the flag name with -- prefix
+								transferArgs = append(transferArgs, "--"+flag.Name)
+							} else if flag.DefaultValue != "" {
+								// For flags with default values, use the default with -- prefix
+								transferArgs = append(transferArgs, "--"+flag.Name, flag.DefaultValue)
+							}
+							s.log.LogDebug("Added flag %s for job %d, config %d", flag.Name, job.ID, config.ID)
+						} else {
+							s.log.LogError("Failed to get rclone flag with ID %d: %v", flagID, err)
+						}
+					}
+				} else {
+					s.log.LogError("Failed to unmarshal command flags for job %d, config %d: %v", job.ID, config.ID, err)
+				}
+			}
+
 			// Source and destination paths
 			var sourcePath, destPath string
 
@@ -985,7 +1036,19 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig,
 
 	// Send webhook notification for success or with errors
 	s.sendWebhookNotification(&job, history, &config)
+}
 
+// isDirectoryBasedTransfer checks if a transfer command operates on directories rather than individual files
+func isDirectoryBasedTransfer(commandName string) bool {
+	// These commands operate on entire directories, not file-by-file
+	dirBasedCommands := map[string]bool{
+		"sync":   true,
+		"bisync": true,
+		"copy":   true,
+		"move":   true,
+	}
+
+	return dirBasedCommands[commandName]
 }
 
 // ProcessOutputPattern processes an output pattern with variables and returns the result
@@ -1788,3 +1851,291 @@ func (s *Scheduler) createJobNotification(job *db.Job, history *db.JobHistory) e
 		message,
 	)
 }
+
+// determineCommandType categorizes rclone commands into types for execution
+func determineCommandType(commandName string) string {
+	// File transfer commands
+	transferCommands := map[string]bool{
+		"copy":   true,
+		"copyto": true,
+		"move":   true,
+		"moveto": true,
+		"sync":   true,
+		"bisync": true,
+	}
+
+	// Listing commands
+	listingCommands := map[string]bool{
+		"ls":          true,
+		"lsd":         true,
+		"lsl":         true,
+		"lsf":         true,
+		"lsjson":      true,
+		"listremotes": true,
+	}
+
+	// Information commands
+	infoCommands := map[string]bool{
+		"md5sum":  true,
+		"sha1sum": true,
+		"size":    true,
+		"version": true,
+	}
+
+	// Directory operations
+	dirCommands := map[string]bool{
+		"mkdir":  true,
+		"rmdir":  true,
+		"rmdirs": true,
+	}
+
+	// Destructive commands
+	destructiveCommands := map[string]bool{
+		"delete": true,
+		"purge":  true,
+	}
+
+	// Maintenance commands
+	maintenanceCommands := map[string]bool{
+		"cleanup": true,
+		"dedupe":  true,
+		"check":   true,
+	}
+
+	// Specialized commands
+	specialCommands := map[string]bool{
+		"obscure":    true,
+		"cryptcheck": true,
+	}
+
+	// Determine the command type
+	if transferCommands[commandName] {
+		return "transfer"
+	} else if listingCommands[commandName] {
+		return "listing"
+	} else if infoCommands[commandName] {
+		return "info"
+	} else if dirCommands[commandName] {
+		return "directory"
+	} else if destructiveCommands[commandName] {
+		return "destructive"
+	} else if maintenanceCommands[commandName] {
+		return "maintenance"
+	} else if specialCommands[commandName] {
+		return "special"
+	}
+
+	// Default to transfer if unknown
+	return "transfer"
+}
+
+// executeSimpleCommand executes rclone commands that don't require file-by-file processing
+func (s *Scheduler) executeSimpleCommand(cmdName string, cmdType string, job db.Job, config db.TransferConfig, history *db.JobHistory, configPath string) {
+	s.log.LogInfo("Executing simple command '%s' of type '%s' for job %d, config %d", cmdName, cmdType, job.ID, config.ID)
+
+	// Prepare base arguments
+	baseArgs := []string{
+		"--config", configPath,
+		"--progress",
+		"--stats-one-line",
+		"--verbose",
+		"--stats", "1s",
+	}
+
+	// Add the command name
+	args := append(baseArgs, cmdName)
+
+	// Add command flags if specified
+	if config.CommandFlags != "" {
+		var flagIDs []uint
+		if err := json.Unmarshal([]byte(config.CommandFlags), &flagIDs); err == nil {
+			// Get the flags for the selected command
+			for _, flagID := range flagIDs {
+				flag, err := s.db.GetRcloneCommandFlag(flagID)
+				if err == nil && flag != nil {
+					if flag.DataType == "bool" {
+						// For boolean flags, just add the flag name with -- prefix
+						args = append(args, "--"+flag.Name)
+					} else if flag.DefaultValue != "" {
+						// For flags with default values, use the default with -- prefix
+						args = append(args, "--"+flag.Name, flag.DefaultValue)
+					}
+					s.log.LogDebug("Added flag %s for job %d, config %d", flag.Name, job.ID, config.ID)
+				} else {
+					s.log.LogError("Failed to get rclone flag with ID %d: %v", flagID, err)
+				}
+			}
+		} else {
+			s.log.LogError("Failed to unmarshal command flags for job %d, config %d: %v", job.ID, config.ID, err)
+		}
+	}
+
+	// Add custom flags if specified
+	if config.RcloneFlags != "" {
+		customFlags := strings.Split(config.RcloneFlags, " ")
+		args = append(args, customFlags...)
+		s.log.LogDebug("Added custom flags for job %d, config %d: %v", job.ID, config.ID, customFlags)
+	}
+
+	// Prepare source and destination paths
+	var sourcePath, destPath string
+
+	// Handle source path with bucket for S3-compatible storage
+	if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
+		if config.SourcePath != "" && config.SourcePath != "/" {
+			sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
+		}
+	} else {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
+	}
+
+	// Handle destination path with bucket for S3-compatible storage
+	if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
+		destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestBucket)
+		if config.DestinationPath != "" && config.DestinationPath != "/" {
+			destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, config.DestinationPath)
+		}
+	} else {
+		destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestinationPath)
+	}
+
+	// Add appropriate paths based on command type
+	switch cmdType {
+	case "transfer":
+		// Directory-based transfers and file-specific transfers handled here
+		args = append(args, sourcePath, destPath)
+	case "maintenance":
+		// Check command needs both source and destination, others may just need source
+		if cmdName == "check" {
+			args = append(args, sourcePath, destPath)
+		} else {
+			args = append(args, sourcePath)
+		}
+	case "listing":
+		// Listing commands only need source path
+		args = append(args, sourcePath)
+	case "info":
+		// Info commands typically need only source path
+		args = append(args, sourcePath)
+	case "directory":
+		// Directory operations might need one or both paths depending on operation
+		if cmdName == "rmdirs" && strings.Contains(config.RcloneFlags, "--dst") {
+			// Special case: rmdirs with --dst flag needs both paths
+			args = append(args, sourcePath, destPath)
+		} else {
+			// Default case: just source path
+			args = append(args, sourcePath)
+		}
+	case "destructive":
+		// Destructive commands only need source path
+		args = append(args, sourcePath)
+	case "special":
+		// Special commands handled case by case
+		if cmdName == "cryptcheck" {
+			args = append(args, sourcePath, destPath)
+		} else if cmdName == "obscure" || cmdName == "version" || cmdName == "listremotes" {
+			// These commands don't need paths at all
+		} else {
+			args = append(args, sourcePath)
+		}
+	default:
+		// Default to source path only
+		args = append(args, sourcePath)
+	}
+
+	// Execute the command
+	rclonePath := os.Getenv("RCLONE_PATH")
+	if rclonePath == "" {
+		rclonePath = "rclone"
+	}
+
+	s.log.LogDebug("Full command: %s %v", rclonePath, args)
+	cmd := exec.Command(rclonePath, args...)
+
+	// Capture output
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+
+	// Start timer for operation
+	startTime := time.Now()
+
+	// Run the command
+	err := cmd.Run()
+
+	// Calculate duration
+	duration := time.Since(startTime)
+
+	// Update history with basic info
+	history.EndTime = &time.Time{}
+	*history.EndTime = startTime.Add(duration)
+
+	// Check for pattern in stderr that indicates successful completion with warnings
+	// Some commands like sync may complete successfully but with warnings
+	successWithWarnings := strings.Contains(stderr.String(), "Transferred:") &&
+		strings.Contains(stderr.String(), "Errors:") &&
+		strings.Contains(stderr.String(), "Checks:")
+
+	// Process results
+	if err != nil && !successWithWarnings {
+		s.log.LogError("Error executing command '%s' for job %d, config %d: %v", cmdName, job.ID, config.ID, err)
+		s.log.LogError("Command stderr: %s", stderr.String())
+
+		history.Status = "failed"
+		history.ErrorMessage = fmt.Sprintf("Command Error: %v\nStderr: %s", err, stderr.String())
+	} else {
+		s.log.LogInfo("Successfully executed command '%s' for job %d, config %d (duration: %v)",
+			cmdName, job.ID, config.ID, duration)
+
+		// Handle different command output types
+		if cmdType == "listing" {
+			// For listing commands, count the number of lines in the output as "files processed"
+			lines := strings.Count(stdout.String(), "\n")
+			history.FilesTransferred = lines
+			history.Status = "completed"
+		} else if cmdType == "transfer" {
+			// Try to extract transfer statistics from command output
+			history.Status = "completed"
+
+			// Look for metrics in stderr which is where rclone puts stats
+			// Extract bytes transferred if available
+			bytesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Bytes`)
+			if matches := bytesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 {
+				if bytesTransferred, err := strconv.ParseInt(matches[1], 10, 64); err == nil {
+					history.BytesTransferred = bytesTransferred
+				}
+			}
+
+			// Extract files transferred if available
+			filesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Files`)
+			if matches := filesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 {
+				if filesTransferred, err := strconv.Atoi(matches[1]); err == nil {
+					history.FilesTransferred = filesTransferred
+				}
+			}
+		} else {
+			// For other commands, we don't have file counts, but the command completed
+			history.Status = "completed"
+		}
+
+		// Store command output in the history for reference
+		if cmdType == "listing" || cmdType == "info" {
+			// For listing and info commands, the output is the result
+			// Limit to first 1000 characters to avoid huge entries
+			output := stdout.String()
+			if len(output) > 1000 {
+				output = output[:997] + "..."
+			}
+			history.ErrorMessage = fmt.Sprintf("Command Output:\n%s", output)
+		}
+	}
+
+	// Update job history in the database
+	if err := s.db.UpdateJobHistory(history); err != nil {
+		s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
+	}
+
+	// Send webhook notification
+	s.sendWebhookNotification(&job, history, &config)
+}

internal/web/handlers/config_handlers.go

@@ -1,9 +1,11 @@
 package handlers
 
 import (
+	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
+	"strconv"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -120,6 +122,41 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) {
 	sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true"
 	config.SourceIncludeArchived = &sourceIncludeArchivedValue
 
+	// Get command_id and validate it
+	commandIDStr := c.Request.FormValue("command_id")
+	if commandIDStr != "" {
+		commandID, err := strconv.ParseUint(commandIDStr, 10, 64)
+		if err != nil {
+			log.Printf("Error parsing command ID: %v", err)
+		} else {
+			config.CommandID = uint(commandID)
+		}
+	} else {
+		// Default to copy command (ID 1)
+		config.CommandID = 1
+	}
+
+	// Get command_flags and store as JSON
+	commandFlags := c.PostFormArray("command_flags")
+	if len(commandFlags) > 0 {
+		flagIDs := make([]uint, 0, len(commandFlags))
+		for _, flagStr := range commandFlags {
+			flagID, err := strconv.ParseUint(flagStr, 10, 64)
+			if err != nil {
+				log.Printf("Error parsing flag ID: %v", err)
+				continue
+			}
+			flagIDs = append(flagIDs, uint(flagID))
+		}
+		flagsJSON, err := json.Marshal(flagIDs)
+		if err != nil {
+			log.Printf("Error marshaling flag IDs: %v", err)
+		} else {
+			config.CommandFlags = string(flagsJSON)
+		}
+	}
+
+	// Process builtin auth settings
 	useBuiltinAuthSourceVal := c.Request.FormValue("use_builtin_auth_source")
 	useBuiltinAuthSourceValue := useBuiltinAuthSourceVal == "on" || useBuiltinAuthSourceVal == "true"
 	config.UseBuiltinAuthSource = &useBuiltinAuthSourceValue
@@ -191,38 +228,49 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) {
 	c.Redirect(http.StatusFound, "/configs")
 }
 
-// HandleUpdateConfig handles the PUT /configs/:id route
+// HandleUpdateConfig handles the POST /configs/:id route
 func (h *Handlers) HandleUpdateConfig(c *gin.Context) {
-	id := c.Param("id")
+	idStr := c.Param("id")
-	userID := c.GetUint("userID")
+	id, err := strconv.ParseUint(idStr, 10, 64)
-
+	if err != nil {
-	var config db.TransferConfig
+		c.String(http.StatusBadRequest, fmt.Sprintf("Invalid ID: %v", err))
-	if err := h.DB.First(&config, id).Error; err != nil {
-		log.Printf("Error finding config: %v", err)
-		c.String(http.StatusNotFound, "Config not found")
 		return
 	}
 
-	// Check if user owns this config
+	existingConfig, err := h.DB.GetTransferConfig(uint(id))
-	if config.CreatedBy != userID {
+	if err != nil {
-		// Check if user is admin
+		c.String(http.StatusInternalServerError, fmt.Sprintf("Error getting config: %v", err))
+		return
+	}
+
+	if existingConfig == nil {
+		c.String(http.StatusNotFound, "Configuration not found")
+		return
+	}
+
+	// Check if the user has permission to edit this config
+	userID := c.GetUint("userID")
+	if existingConfig.CreatedBy != userID {
 		isAdmin, exists := c.Get("isAdmin")
 		if !exists || isAdmin != true {
-			c.String(http.StatusForbidden, "You do not have permission to update this config")
+			c.String(http.StatusForbidden, "You don't have permission to edit this configuration")
 			return
 		}
 	}
 
-	// Get the old config values for comparison
+	var config db.TransferConfig
-	oldConfig := config
 
-	// Bind form data to config
 	if err := c.ShouldBind(&config); err != nil {
 		log.Printf("Error binding config form: %v", err)
 		c.String(http.StatusBadRequest, fmt.Sprintf("Invalid form data: %v", err))
 		return
 	}
 
+	// Preserve the original creator ID and creation time
+	config.ID = existingConfig.ID
+	config.CreatedBy = existingConfig.CreatedBy
+	config.CreatedAt = existingConfig.CreatedAt
+
 	// Process Boolean fields
 	skipProcessedVal := c.Request.FormValue("skip_processed_files")
 	skipProcessedValue := skipProcessedVal == "on" || skipProcessedVal == "true"
@@ -261,6 +309,41 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) {
 	sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true"
 	config.SourceIncludeArchived = &sourceIncludeArchivedValue
 
+	// Get command_id and validate it
+	commandIDStr := c.Request.FormValue("command_id")
+	if commandIDStr != "" {
+		commandID, err := strconv.ParseUint(commandIDStr, 10, 64)
+		if err != nil {
+			log.Printf("Error parsing command ID: %v", err)
+		} else {
+			config.CommandID = uint(commandID)
+		}
+	} else {
+		// Default to copy command (ID 1)
+		config.CommandID = 1
+	}
+
+	// Get command_flags and store as JSON
+	commandFlags := c.PostFormArray("command_flags")
+	if len(commandFlags) > 0 {
+		flagIDs := make([]uint, 0, len(commandFlags))
+		for _, flagStr := range commandFlags {
+			flagID, err := strconv.ParseUint(flagStr, 10, 64)
+			if err != nil {
+				log.Printf("Error parsing flag ID: %v", err)
+				continue
+			}
+			flagIDs = append(flagIDs, uint(flagID))
+		}
+		flagsJSON, err := json.Marshal(flagIDs)
+		if err != nil {
+			log.Printf("Error marshaling flag IDs: %v", err)
+		} else {
+			config.CommandFlags = string(flagsJSON)
+		}
+	}
+
+	// Process builtin auth settings
 	useBuiltinAuthSourceVal := c.Request.FormValue("use_builtin_auth_source")
 	useBuiltinAuthSourceValue := useBuiltinAuthSourceVal == "on" || useBuiltinAuthSourceVal == "true"
 	config.UseBuiltinAuthSource = &useBuiltinAuthSourceValue
@@ -269,82 +352,19 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) {
 	useBuiltinAuthDestValue := useBuiltinAuthDestVal == "on" || useBuiltinAuthDestVal == "true"
 	config.UseBuiltinAuthDest = &useBuiltinAuthDestValue
 
-	// Preserve fields that shouldn't be updated
+	// Preserve the Google Drive authentication status if it's already authenticated
-	config.CreatedBy = oldConfig.CreatedBy
+	config.GoogleDriveAuthenticated = existingConfig.GoogleDriveAuthenticated
 
-	// Start a transaction
+	// Update the LastUpdated timestamp
-	tx := h.DB.Begin()
+	config.UpdatedAt = time.Now()
-	if tx.Error != nil {
+
-		log.Printf("Error beginning transaction: %v", tx.Error)
+	if err := h.DB.UpdateTransferConfig(&config); err != nil {
-		c.String(http.StatusInternalServerError, "Failed to begin transaction")
+		c.String(http.StatusInternalServerError, fmt.Sprintf("Error updating configuration: %v", err))
 		return
 	}
 
-	if err := tx.Save(&config).Error; err != nil {
+	// Redirect to the configs page
-		tx.Rollback()
+	c.Redirect(http.StatusSeeOther, "/configs")
-		log.Printf("Error updating config: %v", err)
-		c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to update config: %v", err))
-		return
-	}
-
-	// Create audit log entry
-	auditDetails := map[string]interface{}{
-		"name":                  config.Name,
-		"source_type":           config.SourceType,
-		"dest_type":             config.DestinationType,
-		"source_path":           config.SourcePath,
-		"dest_path":             config.DestinationPath,
-		"skip_processed_files":  *config.SkipProcessedFiles,
-		"archive_enabled":       *config.ArchiveEnabled,
-		"delete_after_transfer": *config.DeleteAfterTransfer,
-		"source_passive_mode":   *config.SourcePassiveMode,
-		"dest_passive_mode":     *config.DestPassiveMode,
-		"previous_state": map[string]interface{}{
-			"name":                  oldConfig.Name,
-			"source_type":           oldConfig.SourceType,
-			"dest_type":             oldConfig.DestinationType,
-			"source_path":           oldConfig.SourcePath,
-			"dest_path":             oldConfig.DestinationPath,
-			"skip_processed_files":  *oldConfig.SkipProcessedFiles,
-			"archive_enabled":       *oldConfig.ArchiveEnabled,
-			"delete_after_transfer": *oldConfig.DeleteAfterTransfer,
-			"source_passive_mode":   *oldConfig.SourcePassiveMode,
-			"dest_passive_mode":     *oldConfig.DestPassiveMode,
-		},
-	}
-
-	auditLog := db.AuditLog{
-		Action:     "update",
-		EntityType: "config",
-		EntityID:   config.ID,
-		UserID:     userID,
-		Details:    auditDetails,
-		Timestamp:  time.Now(),
-	}
-
-	if err := tx.Create(&auditLog).Error; err != nil {
-		tx.Rollback()
-		log.Printf("Error creating audit log: %v", err)
-		c.String(http.StatusInternalServerError, "Failed to create audit log")
-		return
-	}
-
-	// Commit the transaction
-	if err := tx.Commit().Error; err != nil {
-		log.Printf("Error committing transaction: %v", err)
-		c.String(http.StatusInternalServerError, "Failed to commit transaction")
-		return
-	}
-
-	// Regenerate rclone config file
-	if err := h.DB.GenerateRcloneConfig(&config); err != nil {
-		log.Printf("Warning: Failed to regenerate rclone config: %v", err)
-		// Continue anyway, as the config was updated in the database
-	} else {
-		log.Printf("Regenerated rclone config for config ID %d", config.ID)
-	}
-
-	c.Redirect(http.StatusFound, "/configs")
 }
 
 // HandleDeleteConfig handles the DELETE /configs/:id route

internal/web/handlers/rclone_handlers.go

@@ -0,0 +1,107 @@
+package handlers
+
+import (
+	"html/template"
+	"log"
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/starfleetcptn/gomft/components/providers/common"
+	"github.com/starfleetcptn/gomft/internal/db"
+)
+
+// RcloneHandler contains handlers for rclone-related routes
+type RcloneHandler struct {
+	DB *db.DB
+}
+
+// NewRcloneHandler creates a new RcloneHandler
+func NewRcloneHandler(db *db.DB) *RcloneHandler {
+	return &RcloneHandler{
+		DB: db,
+	}
+}
+
+// RcloneCommandOptions renders the rclone command options for the config form
+func (h *RcloneHandler) RcloneCommandOptions(c *gin.Context) {
+	commands, err := h.DB.GetRcloneCommands()
+	if err != nil {
+		log.Printf("Error getting rclone commands: %v", err)
+		c.String(http.StatusInternalServerError, "Error getting rclone commands")
+		return
+	}
+
+	// Group commands by category
+	categories, err := h.DB.GetRcloneCategories()
+	if err != nil {
+		log.Printf("Error getting rclone categories: %v", err)
+		c.String(http.StatusInternalServerError, "Error getting rclone categories")
+		return
+	}
+
+	// Create a map of category -> commands
+	categoryMap := make(map[string][]db.RcloneCommand)
+	for _, cmd := range commands {
+		categoryMap[cmd.Category] = append(categoryMap[cmd.Category], cmd)
+	}
+
+	_ = common.RcloneCommandOptionsContent(categoryMap, categories).Render(c.Request.Context(), c.Writer)
+}
+
+// RcloneCommandFlags renders the rclone command flags for the selected command
+func (h *RcloneHandler) RcloneCommandFlags(c *gin.Context) {
+	commandIDStr := c.DefaultQuery("command_id", "")
+	if commandIDStr == "" {
+		c.String(http.StatusBadRequest, "Command ID is required")
+		return
+	}
+
+	commandID, err := strconv.ParseUint(commandIDStr, 10, 64)
+	if err != nil {
+		log.Printf("Error parsing command ID: %v", err)
+		c.String(http.StatusBadRequest, "Invalid command ID")
+		return
+	}
+
+	command, err := h.DB.GetRcloneCommandWithFlags(uint(commandID))
+	if err != nil {
+		log.Printf("Error getting rclone command flags: %v", err)
+		c.String(http.StatusInternalServerError, "Error getting rclone command flags")
+		return
+	}
+
+	if command == nil {
+		c.String(http.StatusNotFound, "Command not found")
+		return
+	}
+
+	_ = common.RcloneCommandFlagsContent(command).Render(c.Request.Context(), c.Writer)
+}
+
+// RcloneCommandUsage renders the usage information for a command
+func (h *RcloneHandler) RcloneCommandUsage(c *gin.Context) {
+	commandIDStr := c.Param("id")
+	if commandIDStr == "" {
+		c.String(http.StatusBadRequest, "Command ID is required")
+		return
+	}
+
+	commandID, err := strconv.ParseUint(commandIDStr, 10, 64)
+	if err != nil {
+		log.Printf("Error parsing command ID: %v", err)
+		c.String(http.StatusBadRequest, "Invalid command ID")
+		return
+	}
+
+	usage, err := h.DB.GetRcloneCommandUsage(uint(commandID))
+	if err != nil {
+		log.Printf("Error getting rclone command usage: %v", err)
+		c.String(http.StatusInternalServerError, "Error getting rclone command usage")
+		return
+	}
+
+	c.HTML(http.StatusOK, "command_usage.html", gin.H{
+		"Usage": template.HTML(usage),
+	})
+}

internal/web/handlers/routes.go

@@ -61,6 +61,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		authorized.GET("/configs/gdrive-callback", h.HandleGDriveAuthCallback)
 		authorized.GET("/configs/gdrive-token", h.HandleGDriveTokenProcess)
 
+		// Duplicate rclone endpoints for web interface to access - same path as API
+		rcloneHandler := NewRcloneHandler(h.DB)
+		authorized.GET("api/rclone/commands", func(c *gin.Context) { rcloneHandler.RcloneCommandOptions(c) })
+		authorized.GET("api/rclone/command-flags", func(c *gin.Context) { rcloneHandler.RcloneCommandFlags(c) })
+		authorized.GET("api/rclone/command/:id/usage", func(c *gin.Context) { rcloneHandler.RcloneCommandUsage(c) })
+
 		authorized.GET("/jobs", h.HandleJobs)
 		authorized.GET("/jobs/new", h.HandleNewJob)
 		authorized.GET("/jobs/:id", h.HandleEditJob)