diff --git a/components/config_form.templ b/components/config_form.templ index ebeb3e7..f480858 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -83,6 +83,8 @@ func getInitialData(config *db.TransferConfig) string { skipProcessedFiles := true maxConcurrentTransfers := 4 rcloneFlags := "" + commandId := uint(1) // Default to 'copy' command + commandFlags := "" useBuiltinAuthSource := true useBuiltinAuthDest := true @@ -162,6 +164,8 @@ func getInitialData(config *db.TransferConfig) string { skipProcessedFiles = config.GetSkipProcessedFiles() maxConcurrentTransfers = config.MaxConcurrentTransfers rcloneFlags = config.RcloneFlags + commandId = config.CommandID + commandFlags = config.CommandFlags if config.UseBuiltinAuthSource != nil { useBuiltinAuthSource = *config.UseBuiltinAuthSource } else if sourceClientId != "" || sourceClientSecret != "" { @@ -237,6 +241,8 @@ func getInitialData(config *db.TransferConfig) string { skipProcessedFiles: %v, maxConcurrentTransfers: %d, rcloneFlags: '%s', + commandId: %d, + commandFlags: '%s', // Path validation states sourcePathValid: null, @@ -274,7 +280,8 @@ func getInitialData(config *db.TransferConfig) string { destClientId, destClientSecret, destDriveId, destTeamDrive, destReadOnly, destStartYear, destIncludeArchived, useBuiltinAuthSource, useBuiltinAuthDest, - archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags) + archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags, + commandId, commandFlags) } templ ConfigForm(ctx context.Context, data ConfigFormData) { diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index 94f4927..3c11d44 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -1,5 +1,10 @@ package common +import ( + "fmt" + "github.com/starfleetcptn/gomft/internal/db" +) + templ NameField() {
@@ -135,7 +140,21 @@ templ ArchiveOptions() { templ RcloneFlags() {
- + +
+
+ +
+ @RcloneCommandOptions() +
+

+ Select the rclone command to use for this configuration. +

+ + +
+ +
@@ -150,6 +169,18 @@ templ RcloneFlags() {
} +// New placeholder templ for rclone command options +templ RcloneCommandOptions() { +
+ + +
+} + templ SourceSelection() {
@@ -196,4 +227,80 @@ templ DestinationSelection() {
+} + +// RcloneCommandOptionsContent renders the command options organized by category +templ RcloneCommandOptionsContent(categoryMap map[string][]db.RcloneCommand, categories []string) { + +} + +// RcloneCommandFlagsContent renders the command flags for a selected command +templ RcloneCommandFlagsContent(command *db.RcloneCommand) { + if command == nil { +
Command not found
+ return + } + +
+

{ command.Name }

+

{ command.Description }

+ + if len(command.Flags) > 0 { +
+
Command Flags:
+
+ for _, flag := range command.Flags { +
+ +
+ + if flag.DefaultValue != "" { +

Default: { flag.DefaultValue }

+ } +
+
+ } +
+
+ +
+
+ + Usage Example +
+ rclone { command.Name } [flags] source:path dest:path +
+ } else { +
+
+ + This command doesn't have any specific flags. +
+
+ } +
} \ No newline at end of file diff --git a/internal/db/db.go b/internal/db/db.go index 15de97f..624f867 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -120,9 +120,12 @@ type TransferConfig struct { UseBuiltinAuthDest *bool `form:"use_builtin_auth_dest"` // For Google and other OAuth services GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed // General fields - ArchivePath string `form:"archive_path"` - ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"` - RcloneFlags string `form:"rclone_flags"` + ArchivePath string `form:"archive_path"` + ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"` + RcloneFlags string `form:"rclone_flags"` + // Rclone command fields + CommandID uint `gorm:"default:1" form:"command_id"` // Default to 'copy' command ID (1) + CommandFlags string `form:"command_flags"` // JSON string of selected flags DeleteAfterTransfer *bool `gorm:"default:false" form:"delete_after_transfer"` SkipProcessedFiles *bool `gorm:"default:true" form:"skip_processed_files"` MaxConcurrentTransfers int `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers @@ -250,6 +253,31 @@ type DB struct { *gorm.DB } +// RcloneCommand represents a command available in rclone +type RcloneCommand struct { + ID uint `gorm:"primarykey"` + Name string `gorm:"not null;uniqueIndex"` + Description string `gorm:"not null"` + Category string `gorm:"not null;index"` + IsAdvanced bool `gorm:"not null;default:false"` + Flags []RcloneCommandFlag `gorm:"foreignKey:CommandID;constraint:OnDelete:CASCADE"` + CreatedAt time.Time `gorm:"not null"` +} + +// RcloneCommandFlag represents a flag that can be used with an rclone command +type RcloneCommandFlag struct { + ID uint `gorm:"primarykey"` + CommandID uint `gorm:"not null;index"` + Command RcloneCommand `gorm:"foreignKey:CommandID"` + Name string `gorm:"not null;index"` + ShortName string + Description string `gorm:"not null"` + DataType string `gorm:"not null"` // string, int, bool, etc. + IsRequired bool `gorm:"not null;default:false"` + DefaultValue string + CreatedAt time.Time `gorm:"not null"` +} + func Initialize(dbPath string) (*DB, error) { // Create directory if it doesn't exist dir := filepath.Dir(dbPath) @@ -1707,3 +1735,346 @@ func (u *User) CheckPassword(password string) bool { err := bcrypt.CompareHashAndPassword([]byte(u.PasswordHash), []byte(password)) return err == nil } + +// GetRcloneCommands returns all rclone commands +func (db *DB) GetRcloneCommands() ([]RcloneCommand, error) { + var commands []RcloneCommand + err := db.Find(&commands).Error + return commands, err +} + +// GetRcloneCommand returns a specific rclone command by ID +func (db *DB) GetRcloneCommand(id uint) (*RcloneCommand, error) { + var command RcloneCommand + err := db.First(&command, id).Error + if err != nil { + return nil, err + } + return &command, nil +} + +// GetRcloneCommandByName returns a specific rclone command by name +func (db *DB) GetRcloneCommandByName(name string) (*RcloneCommand, error) { + var command RcloneCommand + err := db.Where("name = ?", name).First(&command).Error + if err != nil { + return nil, err + } + return &command, nil +} + +// GetRcloneCommandsInCategory returns all commands in a specific category +func (db *DB) GetRcloneCommandsInCategory(category string) ([]RcloneCommand, error) { + var commands []RcloneCommand + err := db.Where("category = ?", category).Find(&commands).Error + return commands, err +} + +// GetRcloneCommandFlag returns a specific flag by ID +func (db *DB) GetRcloneCommandFlag(id uint) (*RcloneCommandFlag, error) { + var flag RcloneCommandFlag + err := db.First(&flag, id).Error + if err != nil { + return nil, err + } + return &flag, nil +} + +// GetRcloneCommandFlagByName returns a specific flag by name for a command +func (db *DB) GetRcloneCommandFlagByName(commandID uint, name string) (*RcloneCommandFlag, error) { + var flag RcloneCommandFlag + err := db.Where("command_id = ? AND name = ?", commandID, name).First(&flag).Error + if err != nil { + return nil, err + } + return &flag, nil +} + +// GetRcloneCommandFlags returns all flags for a specific command +func (db *DB) GetRcloneCommandFlags(commandID uint) ([]RcloneCommandFlag, error) { + var flags []RcloneCommandFlag + err := db.Where("command_id = ?", commandID).Find(&flags).Error + return flags, err +} + +// GetRcloneCommandWithFlags returns a command with all its flags +func (db *DB) GetRcloneCommandWithFlags(commandID uint) (*RcloneCommand, error) { + var command RcloneCommand + err := db.Preload("Flags").First(&command, commandID).Error + if err != nil { + return nil, err + } + return &command, nil +} + +// BuildRcloneCommand builds an rclone command string with the specified command and flags +func (db *DB) BuildRcloneCommand(commandName string, flags map[string]string) (string, error) { + // Get the command details + command, err := db.GetRcloneCommandByName(commandName) + if err != nil { + return "", fmt.Errorf("command not found: %s", commandName) + } + + // Start building the command string + cmdStr := "rclone " + command.Name + + // Get all flags for this command + allFlags, err := db.GetRcloneCommandFlags(command.ID) + if err != nil { + return "", fmt.Errorf("failed to get flags for command: %v", err) + } + + // Create a map of flag details for easy lookup + flagDetails := make(map[string]RcloneCommandFlag) + for _, f := range allFlags { + flagDetails[f.Name] = f + } + + // Add the flags to the command + for name, value := range flags { + // Check if the flag exists for this command + flag, exists := flagDetails[name] + if !exists { + return "", fmt.Errorf("invalid flag for command %s: %s", commandName, name) + } + + // Handle different flag types + switch flag.DataType { + case "bool": + if value == "true" { + cmdStr += " " + name + } + default: + cmdStr += " " + name + " " + value + } + } + + return cmdStr, nil +} + +// ValidateRcloneFlags validates if the provided flags are valid for the command +func (db *DB) ValidateRcloneFlags(commandName string, flags map[string]string) (bool, map[string]string) { + // Initialize errors map + errors := make(map[string]string) + + // Get the command details + command, err := db.GetRcloneCommandByName(commandName) + if err != nil { + errors["command"] = "Command not found: " + commandName + return false, errors + } + + // Get all flags for this command + allFlags, err := db.GetRcloneCommandFlags(command.ID) + if err != nil { + errors["command"] = "Failed to get flags for command" + return false, errors + } + + // Create a map of flag details for easy lookup + flagDetails := make(map[string]RcloneCommandFlag) + for _, f := range allFlags { + flagDetails[f.Name] = f + } + + // Check each provided flag + for name, value := range flags { + // Check if the flag exists for this command + flag, exists := flagDetails[name] + if !exists { + errors[name] = "Invalid flag for command " + commandName + continue + } + + // Validate the flag value based on data type + switch flag.DataType { + case "int": + if _, err := strconv.Atoi(value); err != nil { + errors[name] = "Value must be an integer" + } + case "float": + if _, err := strconv.ParseFloat(value, 64); err != nil { + errors[name] = "Value must be a number" + } + case "bool": + if value != "true" && value != "false" { + errors[name] = "Value must be true or false" + } + } + } + + // Check for required flags + for _, flag := range allFlags { + if flag.IsRequired { + if _, provided := flags[flag.Name]; !provided { + errors[flag.Name] = "This flag is required" + } + } + } + + return len(errors) == 0, errors +} + +// GetRcloneCategories returns all unique categories of rclone commands +func (db *DB) GetRcloneCategories() ([]string, error) { + var categories []string + err := db.Model(&RcloneCommand{}).Distinct("category").Pluck("category", &categories).Error + return categories, err +} + +// GetRcloneCommandsByAdvanced returns commands filtered by their advanced status +func (db *DB) GetRcloneCommandsByAdvanced(isAdvanced bool) ([]RcloneCommand, error) { + var commands []RcloneCommand + err := db.Where("is_advanced = ?", isAdvanced).Find(&commands).Error + return commands, err +} + +// SearchRcloneCommands searches for commands by name or description +func (db *DB) SearchRcloneCommands(query string) ([]RcloneCommand, error) { + var commands []RcloneCommand + searchQuery := "%" + query + "%" + err := db.Where("name LIKE ? OR description LIKE ?", searchQuery, searchQuery).Find(&commands).Error + return commands, err +} + +// GetRcloneFlagUsage returns a human-readable usage example for a flag +func (flag *RcloneCommandFlag) GetUsageExample() string { + switch flag.DataType { + case "bool": + return flag.Name + case "int": + return fmt.Sprintf("%s=", flag.Name) + case "float": + return fmt.Sprintf("%s=", flag.Name) + case "string": + return fmt.Sprintf("%s=", flag.Name) + default: + return fmt.Sprintf("%s=", flag.Name) + } +} + +// GetRcloneCommandUsage returns a basic usage example for a command with its required flags +func (db *DB) GetRcloneCommandUsage(commandID uint) (string, error) { + command, err := db.GetRcloneCommandWithFlags(commandID) + if err != nil { + return "", err + } + + usage := fmt.Sprintf("rclone %s [flags] ", command.Name) + + // Add basic usage examples for required flags + requiredFlags := []string{} + for _, flag := range command.Flags { + if flag.IsRequired { + requiredFlags = append(requiredFlags, flag.GetUsageExample()) + } + } + + if len(requiredFlags) > 0 { + usage += "\n\nRequired flags:\n " + strings.Join(requiredFlags, "\n ") + } + + return usage, nil +} + +// ParseRcloneFlags parses a string of rclone flags into a map +func ParseRcloneFlags(flagsStr string) map[string]string { + result := make(map[string]string) + if flagsStr == "" { + return result + } + + // Split the flags string by spaces + parts := strings.Fields(flagsStr) + + for i := 0; i < len(parts); i++ { + part := parts[i] + + // Check if it's a flag (starts with --) + if strings.HasPrefix(part, "--") { + // Remove the -- prefix + flagName := part + + // Check if the flag has a value + if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "--") { + // Next part is a value + result[flagName] = parts[i+1] + i++ // Skip the value in the next iteration + } else { + // Flag without value, treat as boolean + result[flagName] = "true" + } + } + } + + return result +} + +// RenderRcloneCommandHelp generates a help text for a command with its flags +func (db *DB) RenderRcloneCommandHelp(commandID uint) (string, error) { + command, err := db.GetRcloneCommandWithFlags(commandID) + if err != nil { + return "", err + } + + // Build the help text + help := fmt.Sprintf("COMMAND: %s\n", command.Name) + help += fmt.Sprintf("DESCRIPTION: %s\n\n", command.Description) + help += "FLAGS:\n" + + // Group flags by required status + var requiredFlags, optionalFlags []RcloneCommandFlag + for _, flag := range command.Flags { + if flag.IsRequired { + requiredFlags = append(requiredFlags, flag) + } else { + optionalFlags = append(optionalFlags, flag) + } + } + + // Add required flags + if len(requiredFlags) > 0 { + help += " Required:\n" + for _, flag := range requiredFlags { + shortName := "" + if flag.ShortName != "" { + shortName = fmt.Sprintf(" (-%s)", flag.ShortName) + } + help += fmt.Sprintf(" %s%s - %s\n", flag.Name, shortName, flag.Description) + if flag.DataType != "bool" && flag.DefaultValue != "" { + help += fmt.Sprintf(" Default: %s\n", flag.DefaultValue) + } + } + } + + // Add optional flags + if len(optionalFlags) > 0 { + help += "\n Optional:\n" + for _, flag := range optionalFlags { + shortName := "" + if flag.ShortName != "" { + shortName = fmt.Sprintf(" (-%s)", flag.ShortName) + } + help += fmt.Sprintf(" %s%s - %s\n", flag.Name, shortName, flag.Description) + if flag.DataType != "bool" && flag.DefaultValue != "" { + help += fmt.Sprintf(" Default: %s\n", flag.DefaultValue) + } + } + } + + return help, nil +} + +func (db *DB) GetRcloneCommandFlagsMap(commandID uint) (map[uint]RcloneCommandFlag, error) { + flags, err := db.GetRcloneCommandFlags(commandID) + if err != nil { + return nil, err + } + + flagsMap := make(map[uint]RcloneCommandFlag) + for _, flag := range flags { + flagsMap[flag.ID] = flag + } + + return flagsMap, nil +} diff --git a/internal/db/migrations/009_add_rclone_tables.go b/internal/db/migrations/009_add_rclone_tables.go new file mode 100644 index 0000000..a717f8c --- /dev/null +++ b/internal/db/migrations/009_add_rclone_tables.go @@ -0,0 +1,349 @@ +package migrations + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddRcloneTables adds tables for rclone commands and their flags +func AddRcloneTables() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "009_add_rclone_tables", + Migrate: func(tx *gorm.DB) error { + // Check if any tables exist (indicating an existing database) + var count int64 + if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil { + return fmt.Errorf("failed to check for existing tables: %v", err) + } + + // If tables exist, create a backup + if count > 0 { + // Get the database path + sqlDB, err := tx.DB() + if err != nil { + return fmt.Errorf("failed to get underlying database: %v", err) + } + + var seq int + var name, dbPath string + if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil { + return fmt.Errorf("failed to get database path: %v", err) + } + + // Get backup directory from environment variable or use default + backupDir := os.Getenv("BACKUP_DIR") + if backupDir == "" { + backupDir = "/app/backups" // Default Docker path + // Check if we're not in Docker + if _, err := os.Stat(backupDir); os.IsNotExist(err) { + backupDir = "backups" // Fallback to local directory + } + } + + // Create backup directory if it doesn't exist + if err := os.MkdirAll(backupDir, 0755); err != nil { + return fmt.Errorf("failed to create backup directory: %v", err) + } + + // Create backup file with timestamp in the backup directory + dbFileName := filepath.Base(dbPath) + backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405")) + backupFile := filepath.Join(backupDir, backupFileName) + + // Read original database + data, err := os.ReadFile(dbPath) + if err != nil { + return fmt.Errorf("failed to read database for backup: %v", err) + } + + // Write backup + if err := os.WriteFile(backupFile, data, 0600); err != nil { + return fmt.Errorf("failed to create database backup: %v", err) + } + + fmt.Printf("Created database backup at: %s\n", backupFile) + } + + // Create the rclone_commands table + err := tx.Exec(` + CREATE TABLE IF NOT EXISTS rclone_commands ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + description TEXT NOT NULL, + category TEXT NOT NULL, + is_advanced BOOLEAN NOT NULL DEFAULT 0, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `).Error + if err != nil { + return err + } + + // Create an index on name for faster lookups + err = tx.Exec(` + CREATE UNIQUE INDEX IF NOT EXISTS idx_rclone_commands_name ON rclone_commands(name) + `).Error + if err != nil { + return err + } + + // Create an index on category for faster filtering + err = tx.Exec(` + CREATE INDEX IF NOT EXISTS idx_rclone_commands_category ON rclone_commands(category) + `).Error + if err != nil { + return err + } + + // Create the rclone_command_flags table + err = tx.Exec(` + CREATE TABLE IF NOT EXISTS rclone_command_flags ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + command_id INTEGER NOT NULL, + name TEXT NOT NULL, + short_name TEXT, + description TEXT NOT NULL, + data_type TEXT NOT NULL, + is_required BOOLEAN NOT NULL DEFAULT 0, + default_value TEXT, + created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (command_id) REFERENCES rclone_commands(id) ON DELETE CASCADE + ) + `).Error + if err != nil { + return err + } + + // Create index on command_id for faster lookups + err = tx.Exec(` + CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_command_id ON rclone_command_flags(command_id) + `).Error + if err != nil { + return err + } + + // Create index on flag name for faster searches + err = tx.Exec(` + CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_name ON rclone_command_flags(name) + `).Error + if err != nil { + return err + } + + // Populate the tables with default rclone commands and flags + return populateRcloneTablesWithDefaults(tx) + }, + Rollback: func(tx *gorm.DB) error { + // Drop the flags table first due to foreign key constraints + err := tx.Exec(`DROP TABLE IF EXISTS rclone_command_flags`).Error + if err != nil { + return err + } + + // Then drop the commands table + return tx.Exec(`DROP TABLE IF EXISTS rclone_commands`).Error + }, + } +} + +// Helper function to populate rclone tables with default data +func populateRcloneTablesWithDefaults(tx *gorm.DB) error { + // Define base commands + commands := []struct { + Name string + Description string + Category string + IsAdvanced bool + }{ + // Core commands + {"copy", "Copy files from source to dest, skipping already copied", "sync", false}, + {"sync", "Make source and dest identical, modifying destination only", "sync", false}, + {"bisync", "Bidirectional synchronization between two paths", "sync", false}, + {"move", "Move files from source to dest", "sync", false}, + {"delete", "Remove the contents of path", "sync", false}, + {"purge", "Remove the path and all of its contents", "sync", false}, + {"mkdir", "Make the path if it doesn't already exist", "sync", false}, + {"rmdir", "Remove the path", "sync", false}, + {"rmdirs", "Remove any empty directories under the path", "sync", false}, + {"check", "Check if the files in the source and destination match", "sync", false}, + {"ls", "List all the objects in the path with size and path", "listing", false}, + {"lsd", "List all directories/containers/buckets in the path", "listing", false}, + {"lsl", "List all the objects in the path with size, modification time and path", "listing", false}, + {"lsf", "List the objects in the path (obey formatting parameters)", "listing", false}, + {"lsjson", "List the objects in the path in JSON format", "listing", false}, + {"md5sum", "Produce an md5sum file for all the objects in the path", "hash", false}, + {"sha1sum", "Produce a sha1sum file for all the objects in the path", "hash", false}, + {"size", "Return the total size and number of objects in remote:path", "info", false}, + {"version", "Show the version number", "info", false}, + {"cleanup", "Clean up the remote if possible", "maintenance", false}, + {"dedupe", "Interactively find duplicate files and delete/rename them", "maintenance", false}, + {"copyto", "Copy files from source to dest, skipping already copied", "sync", false}, + {"moveto", "Move file or directory from source to dest", "sync", false}, + {"listremotes", "List all the remotes in the config file", "config", false}, + {"obscure", "Obscure password for use in the rclone.conf", "config", false}, + {"cryptcheck", "Check the integrity of an encrypted remote", "crypto", false}, + } + + // Insert commands + for _, cmd := range commands { + err := tx.Exec(` + INSERT INTO rclone_commands + (name, description, category, is_advanced, created_at) + VALUES (?, ?, ?, ?, ?) + `, cmd.Name, cmd.Description, cmd.Category, cmd.IsAdvanced, time.Now()).Error + + if err != nil { + return fmt.Errorf("failed to insert command %s: %v", cmd.Name, err) + } + } + + // Define all flags + allFlags := []struct { + CommandName string + Name string + ShortName string + Description string + DataType string + IsRequired bool + DefaultValue string + }{ + // Global flags - apply to most commands + {"global", "transfers", "n", "Number of file transfers to run in parallel", "int", false, "4"}, + {"global", "checkers", "", "Number of checkers to run in parallel", "int", false, "8"}, + {"global", "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "string", false, "NOTICE"}, + {"global", "stats", "", "Interval between logging stats, e.g 500ms, 60s, 5m", "string", false, "1m0s"}, + {"global", "stats-file-name-length", "", "Max file name length in stats", "int", false, "45"}, + {"global", "stats-one-line", "", "Make the stats fit on one line", "bool", false, "false"}, + {"global", "progress", "p", "Show progress during transfer", "bool", false, "false"}, + {"global", "verbose", "v", "Show verbose output", "bool", false, "false"}, + {"global", "quiet", "q", "Print as little stuff as possible", "bool", false, "false"}, + {"global", "retries", "", "Retry operations this many times if they fail", "int", false, "3"}, + {"global", "retries-sleep", "", "Interval between retrying operations if they fail", "string", false, "0s"}, + {"global", "timeout", "", "IO idle timeout", "string", false, "5m0s"}, + {"global", "tpslimit", "", "Limit HTTP transactions per second", "float", false, "0"}, + {"global", "tpslimit-burst", "", "Max burst of transactions for --tpslimit", "int", false, "1"}, + {"global", "size-only", "", "Skip based on size only, not mod-time or checksum", "bool", false, "false"}, + {"global", "ignore-checksum", "", "Skip post copy check of checksums", "bool", false, "false"}, + {"global", "ignore-existing", "", "Skip all files that exist on destination", "bool", false, "false"}, + {"global", "ignore-size", "", "Skip size checks to calculate if file changed", "bool", false, "false"}, + {"global", "ignore-case-sync", "", "Ignore case when synchronizing", "bool", false, "false"}, + {"global", "no-update-modtime", "", "Don't update destination mod-time if files identical", "bool", false, "false"}, + {"global", "no-check-certificate", "", "Do not verify server SSL certificates", "bool", false, "false"}, + {"global", "ask-password", "", "Allow prompt for password for encrypted config", "bool", false, "false"}, + {"global", "dump", "", "List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles", "string", false, ""}, + {"global", "metadata", "M", "Preserve metadata when copying objects", "bool", false, "false"}, + {"global", "metadata-set", "", "Add metadata key=value when uploading", "string", false, ""}, + + // copy command specific flags + {"copy", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + {"copy", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"}, + {"copy", "cutoff-mode", "", "Mode to stop transfers when reaching the cutoff threshold", "string", false, "hard"}, + {"copy", "max-transfer", "", "Maximum size of data to transfer", "string", false, "off"}, + {"copy", "max-backlog", "", "Maximum size of upload or download backlog", "int", false, "10000"}, + {"copy", "track-renames", "", "Track file renames during sync", "bool", false, "false"}, + {"copy", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"}, + + // sync command specific flags + {"sync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + {"sync", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"}, + {"sync", "backup-dir", "", "Make backups into this directory", "string", false, ""}, + {"sync", "suffix", "", "Suffix to add to changed files", "string", false, ""}, + {"sync", "delete-before", "", "Delete before transferring", "bool", false, "false"}, + {"sync", "delete-during", "", "Delete during transferring", "bool", false, "false"}, + {"sync", "delete-after", "", "Delete after transferring", "bool", false, "false"}, + {"sync", "track-renames", "", "Track file renames during sync", "bool", false, "false"}, + {"sync", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"}, + + // bisync command specific flags + {"bisync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + {"bisync", "resync", "", "Performs the resync run", "bool", false, "false"}, + {"bisync", "check-access", "", "Ensure destination has write access", "bool", false, "true"}, + {"bisync", "conflict-resolve", "", "Automatically resolve conflicts (newer|larger|older|smaller)", "string", false, ""}, + {"bisync", "max-delete", "", "Safety check on maximum files to delete", "int", false, "10"}, + + // move command specific flags + {"move", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + {"move", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"}, + {"move", "delete-empty-src-dirs", "", "Delete empty source dirs after move", "bool", false, "false"}, + + // ls/listing related flags + {"ls", "recursive", "R", "Recurse into the listing", "bool", false, "false"}, + {"ls", "max-depth", "", "Maximum depth to recursively list", "int", false, ""}, + {"ls", "format", "", "Format for the output", "string", false, ""}, + {"ls", "absolute", "", "Put a leading / in front of path names", "bool", false, "false"}, + + {"lsl", "recursive", "R", "Recurse into the listing", "bool", false, "false"}, + {"lsl", "max-depth", "", "Maximum depth to recursively list", "int", false, ""}, + + {"lsd", "max-depth", "", "Maximum depth to show in the listing", "int", false, ""}, + {"lsd", "dir-sort", "", "Directory sorting (alphabetical|size|time)", "string", false, "alphabetical"}, + + {"lsf", "format", "", "Format for the output", "string", false, ""}, + {"lsf", "recursive", "R", "Recurse into the listing", "bool", false, "false"}, + {"lsf", "max-depth", "", "Maximum depth to recursively list", "int", false, ""}, + + {"lsjson", "recursive", "R", "Recurse into the listing", "bool", false, "false"}, + {"lsjson", "max-depth", "", "Maximum depth to recursively list", "int", false, ""}, + {"lsjson", "files-only", "", "Show only files, not directories", "bool", false, "false"}, + {"lsjson", "encrypted", "", "Show the encrypted names", "bool", false, "false"}, + {"lsjson", "stat", "", "Show status of objects", "bool", false, "false"}, + + // Other specialized command flags + {"cryptcheck", "one-way", "", "Check one way only, source files must exist on destination", "bool", false, "false"}, + + {"cleanup", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + + {"dedupe", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"}, + {"dedupe", "mode", "", "Dedupe mode interactive|skip|first|newest|oldest|largest|smallest|rename", "string", false, "interactive"}, + } + + // Insert flags with associated command IDs + for _, flag := range allFlags { + // For global flags, add them to all commands + if flag.CommandName == "global" { + // Get all command IDs + var commandIDs []int64 + err := tx.Raw(`SELECT id FROM rclone_commands`).Scan(&commandIDs).Error + if err != nil { + return fmt.Errorf("failed to get all command IDs: %v", err) + } + + // Add global flag to each command + for _, cmdID := range commandIDs { + err = tx.Exec(` + INSERT INTO rclone_command_flags + (command_id, name, short_name, description, data_type, is_required, default_value, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, cmdID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error + + if err != nil { + return fmt.Errorf("failed to insert global flag %s for command ID %d: %v", flag.Name, cmdID, err) + } + } + } else { + // Add command-specific flag + var commandID int64 + err := tx.Raw(`SELECT id FROM rclone_commands WHERE name = ?`, flag.CommandName).Scan(&commandID).Error + if err != nil { + return fmt.Errorf("failed to find command %s: %v", flag.CommandName, err) + } + + err = tx.Exec(` + INSERT INTO rclone_command_flags + (command_id, name, short_name, description, data_type, is_required, default_value, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + `, commandID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error + + if err != nil { + return fmt.Errorf("failed to insert flag %s for command %s: %v", flag.Name, flag.CommandName, err) + } + } + } + + return nil +} diff --git a/internal/db/migrations/010_add_rclone_command_to_config.go b/internal/db/migrations/010_add_rclone_command_to_config.go new file mode 100644 index 0000000..b1a3c76 --- /dev/null +++ b/internal/db/migrations/010_add_rclone_command_to_config.go @@ -0,0 +1,96 @@ +package migrations + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddRcloneCommandToConfig adds rclone command fields to the transfer_configs table +func AddRcloneCommandToConfig() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "010_add_rclone_command_to_config", + Migrate: func(tx *gorm.DB) error { + // Check if any tables exist (indicating an existing database) + var count int64 + if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil { + return fmt.Errorf("failed to check for existing tables: %v", err) + } + + // If tables exist, create a backup + if count > 0 { + // Get the database path + sqlDB, err := tx.DB() + if err != nil { + return fmt.Errorf("failed to get underlying database: %v", err) + } + + var seq int + var name, dbPath string + if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil { + return fmt.Errorf("failed to get database path: %v", err) + } + + // Get backup directory from environment variable or use default + backupDir := os.Getenv("BACKUP_DIR") + if backupDir == "" { + backupDir = "/app/backups" // Default Docker path + // Check if we're not in Docker + if _, err := os.Stat(backupDir); os.IsNotExist(err) { + backupDir = "backups" // Fallback to local directory + } + } + + // Create backup directory if it doesn't exist + if err := os.MkdirAll(backupDir, 0755); err != nil { + return fmt.Errorf("failed to create backup directory: %v", err) + } + + // Create backup file with timestamp in the backup directory + dbFileName := filepath.Base(dbPath) + backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405")) + backupFile := filepath.Join(backupDir, backupFileName) + + // Read original database + data, err := os.ReadFile(dbPath) + if err != nil { + return fmt.Errorf("failed to read database for backup: %v", err) + } + + // Write backup + if err := os.WriteFile(backupFile, data, 0600); err != nil { + return fmt.Errorf("failed to create database backup: %v", err) + } + + fmt.Printf("Created database backup at: %s\n", backupFile) + } + + // Add the command_id and command_flags columns to the transfer_configs table + if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_id INTEGER DEFAULT NULL REFERENCES rclone_commands(id)`).Error; err != nil { + return err + } + + if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_flags TEXT DEFAULT NULL`).Error; err != nil { + return err + } + + return nil + }, + Rollback: func(tx *gorm.DB) error { + // Remove the columns in reverse order + if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_flags`).Error; err != nil { + return err + } + + if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_id`).Error; err != nil { + return err + } + + return nil + }, + } +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 4c7157c..79980ae 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -19,6 +19,8 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate { AddTimestampsToJobHistories(), // 006 AddNotificationServices(), // 007 AddUserNotifications(), // 008 + AddRcloneTables(), // 009 + AddRcloneCommandToConfig(), // 010 ) return gormigrate.New(db, gormigrate.DefaultOptions, migrations) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index dc4c6e8..f58ca66 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -464,6 +464,30 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, // Get rclone config path configPath := s.db.GetConfigRclonePath(&config) + // Get the command to use for the transfer + var rcloneCommand string = "copyto" // Default command + if config.CommandID > 0 { + // Get the command by ID + command, err := s.db.GetRcloneCommand(config.CommandID) + if err == nil && command != nil { + rcloneCommand = command.Name + s.log.LogDebug("Using rclone command %s for job %d, config %d", rcloneCommand, job.ID, config.ID) + } else { + s.log.LogError("Failed to get rclone command with ID %d: %v", config.CommandID, err) + } + } + + // Determine command type to handle execution appropriately + commandType := determineCommandType(rcloneCommand) + s.log.LogDebug("Command %s is of type: %s", rcloneCommand, commandType) + + // For non-file-by-file transfer commands, use the simple execution approach + if commandType != "transfer" || isDirectoryBasedTransfer(rcloneCommand) { + s.executeSimpleCommand(rcloneCommand, commandType, job, config, history, configPath) + return + } + + // The rest of the function handles file-by-file transfer commands (copyto, moveto) // Use lsjson to get file list and metadata in one operation instead of separate size and ls commands listArgs := []string{ "--config", configPath, @@ -564,7 +588,7 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, var files []map[string]interface{} var totalSize int64 for _, entry := range fileEntries { - // Skip directories + // Process directories if isDir, ok := entry["IsDir"].(bool); ok && isDir { continue } @@ -753,16 +777,43 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, wg.Done() }() - // Prepare moveto command for transfer + // Prepare rclone command transferArgs := []string{ "--config", configPath, - "copyto", "--progress", "--stats-one-line", "--verbose", "--stats", "1s", } + // Add the command to the arguments + transferArgs = append(transferArgs, rcloneCommand) + + // Add command flags if specified + if config.CommandFlags != "" { + var flagIDs []uint + if err := json.Unmarshal([]byte(config.CommandFlags), &flagIDs); err == nil { + // Get the flags for the selected command + for _, flagID := range flagIDs { + flag, err := s.db.GetRcloneCommandFlag(flagID) + if err == nil && flag != nil { + if flag.DataType == "bool" { + // For boolean flags, just add the flag name with -- prefix + transferArgs = append(transferArgs, "--"+flag.Name) + } else if flag.DefaultValue != "" { + // For flags with default values, use the default with -- prefix + transferArgs = append(transferArgs, "--"+flag.Name, flag.DefaultValue) + } + s.log.LogDebug("Added flag %s for job %d, config %d", flag.Name, job.ID, config.ID) + } else { + s.log.LogError("Failed to get rclone flag with ID %d: %v", flagID, err) + } + } + } else { + s.log.LogError("Failed to unmarshal command flags for job %d, config %d: %v", job.ID, config.ID, err) + } + } + // Source and destination paths var sourcePath, destPath string @@ -985,7 +1036,19 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, // Send webhook notification for success or with errors s.sendWebhookNotification(&job, history, &config) +} +// isDirectoryBasedTransfer checks if a transfer command operates on directories rather than individual files +func isDirectoryBasedTransfer(commandName string) bool { + // These commands operate on entire directories, not file-by-file + dirBasedCommands := map[string]bool{ + "sync": true, + "bisync": true, + "copy": true, + "move": true, + } + + return dirBasedCommands[commandName] } // ProcessOutputPattern processes an output pattern with variables and returns the result @@ -1788,3 +1851,291 @@ func (s *Scheduler) createJobNotification(job *db.Job, history *db.JobHistory) e message, ) } + +// determineCommandType categorizes rclone commands into types for execution +func determineCommandType(commandName string) string { + // File transfer commands + transferCommands := map[string]bool{ + "copy": true, + "copyto": true, + "move": true, + "moveto": true, + "sync": true, + "bisync": true, + } + + // Listing commands + listingCommands := map[string]bool{ + "ls": true, + "lsd": true, + "lsl": true, + "lsf": true, + "lsjson": true, + "listremotes": true, + } + + // Information commands + infoCommands := map[string]bool{ + "md5sum": true, + "sha1sum": true, + "size": true, + "version": true, + } + + // Directory operations + dirCommands := map[string]bool{ + "mkdir": true, + "rmdir": true, + "rmdirs": true, + } + + // Destructive commands + destructiveCommands := map[string]bool{ + "delete": true, + "purge": true, + } + + // Maintenance commands + maintenanceCommands := map[string]bool{ + "cleanup": true, + "dedupe": true, + "check": true, + } + + // Specialized commands + specialCommands := map[string]bool{ + "obscure": true, + "cryptcheck": true, + } + + // Determine the command type + if transferCommands[commandName] { + return "transfer" + } else if listingCommands[commandName] { + return "listing" + } else if infoCommands[commandName] { + return "info" + } else if dirCommands[commandName] { + return "directory" + } else if destructiveCommands[commandName] { + return "destructive" + } else if maintenanceCommands[commandName] { + return "maintenance" + } else if specialCommands[commandName] { + return "special" + } + + // Default to transfer if unknown + return "transfer" +} + +// executeSimpleCommand executes rclone commands that don't require file-by-file processing +func (s *Scheduler) executeSimpleCommand(cmdName string, cmdType string, job db.Job, config db.TransferConfig, history *db.JobHistory, configPath string) { + s.log.LogInfo("Executing simple command '%s' of type '%s' for job %d, config %d", cmdName, cmdType, job.ID, config.ID) + + // Prepare base arguments + baseArgs := []string{ + "--config", configPath, + "--progress", + "--stats-one-line", + "--verbose", + "--stats", "1s", + } + + // Add the command name + args := append(baseArgs, cmdName) + + // Add command flags if specified + if config.CommandFlags != "" { + var flagIDs []uint + if err := json.Unmarshal([]byte(config.CommandFlags), &flagIDs); err == nil { + // Get the flags for the selected command + for _, flagID := range flagIDs { + flag, err := s.db.GetRcloneCommandFlag(flagID) + if err == nil && flag != nil { + if flag.DataType == "bool" { + // For boolean flags, just add the flag name with -- prefix + args = append(args, "--"+flag.Name) + } else if flag.DefaultValue != "" { + // For flags with default values, use the default with -- prefix + args = append(args, "--"+flag.Name, flag.DefaultValue) + } + s.log.LogDebug("Added flag %s for job %d, config %d", flag.Name, job.ID, config.ID) + } else { + s.log.LogError("Failed to get rclone flag with ID %d: %v", flagID, err) + } + } + } else { + s.log.LogError("Failed to unmarshal command flags for job %d, config %d: %v", job.ID, config.ID, err) + } + } + + // Add custom flags if specified + if config.RcloneFlags != "" { + customFlags := strings.Split(config.RcloneFlags, " ") + args = append(args, customFlags...) + s.log.LogDebug("Added custom flags for job %d, config %d: %v", job.ID, config.ID, customFlags) + } + + // Prepare source and destination paths + var sourcePath, destPath string + + // Handle source path with bucket for S3-compatible storage + if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" { + sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket) + if config.SourcePath != "" && config.SourcePath != "/" { + sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath) + } + } else { + sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath) + } + + // Handle destination path with bucket for S3-compatible storage + if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" { + destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestBucket) + if config.DestinationPath != "" && config.DestinationPath != "/" { + destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, config.DestinationPath) + } + } else { + destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestinationPath) + } + + // Add appropriate paths based on command type + switch cmdType { + case "transfer": + // Directory-based transfers and file-specific transfers handled here + args = append(args, sourcePath, destPath) + case "maintenance": + // Check command needs both source and destination, others may just need source + if cmdName == "check" { + args = append(args, sourcePath, destPath) + } else { + args = append(args, sourcePath) + } + case "listing": + // Listing commands only need source path + args = append(args, sourcePath) + case "info": + // Info commands typically need only source path + args = append(args, sourcePath) + case "directory": + // Directory operations might need one or both paths depending on operation + if cmdName == "rmdirs" && strings.Contains(config.RcloneFlags, "--dst") { + // Special case: rmdirs with --dst flag needs both paths + args = append(args, sourcePath, destPath) + } else { + // Default case: just source path + args = append(args, sourcePath) + } + case "destructive": + // Destructive commands only need source path + args = append(args, sourcePath) + case "special": + // Special commands handled case by case + if cmdName == "cryptcheck" { + args = append(args, sourcePath, destPath) + } else if cmdName == "obscure" || cmdName == "version" || cmdName == "listremotes" { + // These commands don't need paths at all + } else { + args = append(args, sourcePath) + } + default: + // Default to source path only + args = append(args, sourcePath) + } + + // Execute the command + rclonePath := os.Getenv("RCLONE_PATH") + if rclonePath == "" { + rclonePath = "rclone" + } + + s.log.LogDebug("Full command: %s %v", rclonePath, args) + cmd := exec.Command(rclonePath, args...) + + // Capture output + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + // Start timer for operation + startTime := time.Now() + + // Run the command + err := cmd.Run() + + // Calculate duration + duration := time.Since(startTime) + + // Update history with basic info + history.EndTime = &time.Time{} + *history.EndTime = startTime.Add(duration) + + // Check for pattern in stderr that indicates successful completion with warnings + // Some commands like sync may complete successfully but with warnings + successWithWarnings := strings.Contains(stderr.String(), "Transferred:") && + strings.Contains(stderr.String(), "Errors:") && + strings.Contains(stderr.String(), "Checks:") + + // Process results + if err != nil && !successWithWarnings { + s.log.LogError("Error executing command '%s' for job %d, config %d: %v", cmdName, job.ID, config.ID, err) + s.log.LogError("Command stderr: %s", stderr.String()) + + history.Status = "failed" + history.ErrorMessage = fmt.Sprintf("Command Error: %v\nStderr: %s", err, stderr.String()) + } else { + s.log.LogInfo("Successfully executed command '%s' for job %d, config %d (duration: %v)", + cmdName, job.ID, config.ID, duration) + + // Handle different command output types + if cmdType == "listing" { + // For listing commands, count the number of lines in the output as "files processed" + lines := strings.Count(stdout.String(), "\n") + history.FilesTransferred = lines + history.Status = "completed" + } else if cmdType == "transfer" { + // Try to extract transfer statistics from command output + history.Status = "completed" + + // Look for metrics in stderr which is where rclone puts stats + // Extract bytes transferred if available + bytesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Bytes`) + if matches := bytesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 { + if bytesTransferred, err := strconv.ParseInt(matches[1], 10, 64); err == nil { + history.BytesTransferred = bytesTransferred + } + } + + // Extract files transferred if available + filesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Files`) + if matches := filesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 { + if filesTransferred, err := strconv.Atoi(matches[1]); err == nil { + history.FilesTransferred = filesTransferred + } + } + } else { + // For other commands, we don't have file counts, but the command completed + history.Status = "completed" + } + + // Store command output in the history for reference + if cmdType == "listing" || cmdType == "info" { + // For listing and info commands, the output is the result + // Limit to first 1000 characters to avoid huge entries + output := stdout.String() + if len(output) > 1000 { + output = output[:997] + "..." + } + history.ErrorMessage = fmt.Sprintf("Command Output:\n%s", output) + } + } + + // Update job history in the database + if err := s.db.UpdateJobHistory(history); err != nil { + s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err) + } + + // Send webhook notification + s.sendWebhookNotification(&job, history, &config) +} diff --git a/internal/web/handlers/config_handlers.go b/internal/web/handlers/config_handlers.go index 2c433b2..17180eb 100644 --- a/internal/web/handlers/config_handlers.go +++ b/internal/web/handlers/config_handlers.go @@ -1,9 +1,11 @@ package handlers import ( + "encoding/json" "fmt" "log" "net/http" + "strconv" "time" "github.com/gin-gonic/gin" @@ -120,6 +122,41 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) { sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" config.SourceIncludeArchived = &sourceIncludeArchivedValue + // Get command_id and validate it + commandIDStr := c.Request.FormValue("command_id") + if commandIDStr != "" { + commandID, err := strconv.ParseUint(commandIDStr, 10, 64) + if err != nil { + log.Printf("Error parsing command ID: %v", err) + } else { + config.CommandID = uint(commandID) + } + } else { + // Default to copy command (ID 1) + config.CommandID = 1 + } + + // Get command_flags and store as JSON + commandFlags := c.PostFormArray("command_flags") + if len(commandFlags) > 0 { + flagIDs := make([]uint, 0, len(commandFlags)) + for _, flagStr := range commandFlags { + flagID, err := strconv.ParseUint(flagStr, 10, 64) + if err != nil { + log.Printf("Error parsing flag ID: %v", err) + continue + } + flagIDs = append(flagIDs, uint(flagID)) + } + flagsJSON, err := json.Marshal(flagIDs) + if err != nil { + log.Printf("Error marshaling flag IDs: %v", err) + } else { + config.CommandFlags = string(flagsJSON) + } + } + + // Process builtin auth settings useBuiltinAuthSourceVal := c.Request.FormValue("use_builtin_auth_source") useBuiltinAuthSourceValue := useBuiltinAuthSourceVal == "on" || useBuiltinAuthSourceVal == "true" config.UseBuiltinAuthSource = &useBuiltinAuthSourceValue @@ -191,38 +228,49 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) { c.Redirect(http.StatusFound, "/configs") } -// HandleUpdateConfig handles the PUT /configs/:id route +// HandleUpdateConfig handles the POST /configs/:id route func (h *Handlers) HandleUpdateConfig(c *gin.Context) { - id := c.Param("id") - userID := c.GetUint("userID") - - var config db.TransferConfig - if err := h.DB.First(&config, id).Error; err != nil { - log.Printf("Error finding config: %v", err) - c.String(http.StatusNotFound, "Config not found") + idStr := c.Param("id") + id, err := strconv.ParseUint(idStr, 10, 64) + if err != nil { + c.String(http.StatusBadRequest, fmt.Sprintf("Invalid ID: %v", err)) return } - // Check if user owns this config - if config.CreatedBy != userID { - // Check if user is admin + existingConfig, err := h.DB.GetTransferConfig(uint(id)) + if err != nil { + c.String(http.StatusInternalServerError, fmt.Sprintf("Error getting config: %v", err)) + return + } + + if existingConfig == nil { + c.String(http.StatusNotFound, "Configuration not found") + return + } + + // Check if the user has permission to edit this config + userID := c.GetUint("userID") + if existingConfig.CreatedBy != userID { isAdmin, exists := c.Get("isAdmin") if !exists || isAdmin != true { - c.String(http.StatusForbidden, "You do not have permission to update this config") + c.String(http.StatusForbidden, "You don't have permission to edit this configuration") return } } - // Get the old config values for comparison - oldConfig := config + var config db.TransferConfig - // Bind form data to config if err := c.ShouldBind(&config); err != nil { log.Printf("Error binding config form: %v", err) c.String(http.StatusBadRequest, fmt.Sprintf("Invalid form data: %v", err)) return } + // Preserve the original creator ID and creation time + config.ID = existingConfig.ID + config.CreatedBy = existingConfig.CreatedBy + config.CreatedAt = existingConfig.CreatedAt + // Process Boolean fields skipProcessedVal := c.Request.FormValue("skip_processed_files") skipProcessedValue := skipProcessedVal == "on" || skipProcessedVal == "true" @@ -261,6 +309,41 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) { sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" config.SourceIncludeArchived = &sourceIncludeArchivedValue + // Get command_id and validate it + commandIDStr := c.Request.FormValue("command_id") + if commandIDStr != "" { + commandID, err := strconv.ParseUint(commandIDStr, 10, 64) + if err != nil { + log.Printf("Error parsing command ID: %v", err) + } else { + config.CommandID = uint(commandID) + } + } else { + // Default to copy command (ID 1) + config.CommandID = 1 + } + + // Get command_flags and store as JSON + commandFlags := c.PostFormArray("command_flags") + if len(commandFlags) > 0 { + flagIDs := make([]uint, 0, len(commandFlags)) + for _, flagStr := range commandFlags { + flagID, err := strconv.ParseUint(flagStr, 10, 64) + if err != nil { + log.Printf("Error parsing flag ID: %v", err) + continue + } + flagIDs = append(flagIDs, uint(flagID)) + } + flagsJSON, err := json.Marshal(flagIDs) + if err != nil { + log.Printf("Error marshaling flag IDs: %v", err) + } else { + config.CommandFlags = string(flagsJSON) + } + } + + // Process builtin auth settings useBuiltinAuthSourceVal := c.Request.FormValue("use_builtin_auth_source") useBuiltinAuthSourceValue := useBuiltinAuthSourceVal == "on" || useBuiltinAuthSourceVal == "true" config.UseBuiltinAuthSource = &useBuiltinAuthSourceValue @@ -269,82 +352,19 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) { useBuiltinAuthDestValue := useBuiltinAuthDestVal == "on" || useBuiltinAuthDestVal == "true" config.UseBuiltinAuthDest = &useBuiltinAuthDestValue - // Preserve fields that shouldn't be updated - config.CreatedBy = oldConfig.CreatedBy + // Preserve the Google Drive authentication status if it's already authenticated + config.GoogleDriveAuthenticated = existingConfig.GoogleDriveAuthenticated - // Start a transaction - tx := h.DB.Begin() - if tx.Error != nil { - log.Printf("Error beginning transaction: %v", tx.Error) - c.String(http.StatusInternalServerError, "Failed to begin transaction") + // Update the LastUpdated timestamp + config.UpdatedAt = time.Now() + + if err := h.DB.UpdateTransferConfig(&config); err != nil { + c.String(http.StatusInternalServerError, fmt.Sprintf("Error updating configuration: %v", err)) return } - if err := tx.Save(&config).Error; err != nil { - tx.Rollback() - log.Printf("Error updating config: %v", err) - c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to update config: %v", err)) - return - } - - // Create audit log entry - auditDetails := map[string]interface{}{ - "name": config.Name, - "source_type": config.SourceType, - "dest_type": config.DestinationType, - "source_path": config.SourcePath, - "dest_path": config.DestinationPath, - "skip_processed_files": *config.SkipProcessedFiles, - "archive_enabled": *config.ArchiveEnabled, - "delete_after_transfer": *config.DeleteAfterTransfer, - "source_passive_mode": *config.SourcePassiveMode, - "dest_passive_mode": *config.DestPassiveMode, - "previous_state": map[string]interface{}{ - "name": oldConfig.Name, - "source_type": oldConfig.SourceType, - "dest_type": oldConfig.DestinationType, - "source_path": oldConfig.SourcePath, - "dest_path": oldConfig.DestinationPath, - "skip_processed_files": *oldConfig.SkipProcessedFiles, - "archive_enabled": *oldConfig.ArchiveEnabled, - "delete_after_transfer": *oldConfig.DeleteAfterTransfer, - "source_passive_mode": *oldConfig.SourcePassiveMode, - "dest_passive_mode": *oldConfig.DestPassiveMode, - }, - } - - auditLog := db.AuditLog{ - Action: "update", - EntityType: "config", - EntityID: config.ID, - UserID: userID, - Details: auditDetails, - Timestamp: time.Now(), - } - - if err := tx.Create(&auditLog).Error; err != nil { - tx.Rollback() - log.Printf("Error creating audit log: %v", err) - c.String(http.StatusInternalServerError, "Failed to create audit log") - return - } - - // Commit the transaction - if err := tx.Commit().Error; err != nil { - log.Printf("Error committing transaction: %v", err) - c.String(http.StatusInternalServerError, "Failed to commit transaction") - return - } - - // Regenerate rclone config file - if err := h.DB.GenerateRcloneConfig(&config); err != nil { - log.Printf("Warning: Failed to regenerate rclone config: %v", err) - // Continue anyway, as the config was updated in the database - } else { - log.Printf("Regenerated rclone config for config ID %d", config.ID) - } - - c.Redirect(http.StatusFound, "/configs") + // Redirect to the configs page + c.Redirect(http.StatusSeeOther, "/configs") } // HandleDeleteConfig handles the DELETE /configs/:id route diff --git a/internal/web/handlers/rclone_handlers.go b/internal/web/handlers/rclone_handlers.go new file mode 100644 index 0000000..e1c3afe --- /dev/null +++ b/internal/web/handlers/rclone_handlers.go @@ -0,0 +1,107 @@ +package handlers + +import ( + "html/template" + "log" + "net/http" + "strconv" + + "github.com/gin-gonic/gin" + "github.com/starfleetcptn/gomft/components/providers/common" + "github.com/starfleetcptn/gomft/internal/db" +) + +// RcloneHandler contains handlers for rclone-related routes +type RcloneHandler struct { + DB *db.DB +} + +// NewRcloneHandler creates a new RcloneHandler +func NewRcloneHandler(db *db.DB) *RcloneHandler { + return &RcloneHandler{ + DB: db, + } +} + +// RcloneCommandOptions renders the rclone command options for the config form +func (h *RcloneHandler) RcloneCommandOptions(c *gin.Context) { + commands, err := h.DB.GetRcloneCommands() + if err != nil { + log.Printf("Error getting rclone commands: %v", err) + c.String(http.StatusInternalServerError, "Error getting rclone commands") + return + } + + // Group commands by category + categories, err := h.DB.GetRcloneCategories() + if err != nil { + log.Printf("Error getting rclone categories: %v", err) + c.String(http.StatusInternalServerError, "Error getting rclone categories") + return + } + + // Create a map of category -> commands + categoryMap := make(map[string][]db.RcloneCommand) + for _, cmd := range commands { + categoryMap[cmd.Category] = append(categoryMap[cmd.Category], cmd) + } + + _ = common.RcloneCommandOptionsContent(categoryMap, categories).Render(c.Request.Context(), c.Writer) +} + +// RcloneCommandFlags renders the rclone command flags for the selected command +func (h *RcloneHandler) RcloneCommandFlags(c *gin.Context) { + commandIDStr := c.DefaultQuery("command_id", "") + if commandIDStr == "" { + c.String(http.StatusBadRequest, "Command ID is required") + return + } + + commandID, err := strconv.ParseUint(commandIDStr, 10, 64) + if err != nil { + log.Printf("Error parsing command ID: %v", err) + c.String(http.StatusBadRequest, "Invalid command ID") + return + } + + command, err := h.DB.GetRcloneCommandWithFlags(uint(commandID)) + if err != nil { + log.Printf("Error getting rclone command flags: %v", err) + c.String(http.StatusInternalServerError, "Error getting rclone command flags") + return + } + + if command == nil { + c.String(http.StatusNotFound, "Command not found") + return + } + + _ = common.RcloneCommandFlagsContent(command).Render(c.Request.Context(), c.Writer) +} + +// RcloneCommandUsage renders the usage information for a command +func (h *RcloneHandler) RcloneCommandUsage(c *gin.Context) { + commandIDStr := c.Param("id") + if commandIDStr == "" { + c.String(http.StatusBadRequest, "Command ID is required") + return + } + + commandID, err := strconv.ParseUint(commandIDStr, 10, 64) + if err != nil { + log.Printf("Error parsing command ID: %v", err) + c.String(http.StatusBadRequest, "Invalid command ID") + return + } + + usage, err := h.DB.GetRcloneCommandUsage(uint(commandID)) + if err != nil { + log.Printf("Error getting rclone command usage: %v", err) + c.String(http.StatusInternalServerError, "Error getting rclone command usage") + return + } + + c.HTML(http.StatusOK, "command_usage.html", gin.H{ + "Usage": template.HTML(usage), + }) +} diff --git a/internal/web/handlers/routes.go b/internal/web/handlers/routes.go index 07e2c6a..ad7003d 100644 --- a/internal/web/handlers/routes.go +++ b/internal/web/handlers/routes.go @@ -61,6 +61,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) { authorized.GET("/configs/gdrive-callback", h.HandleGDriveAuthCallback) authorized.GET("/configs/gdrive-token", h.HandleGDriveTokenProcess) + // Duplicate rclone endpoints for web interface to access - same path as API + rcloneHandler := NewRcloneHandler(h.DB) + authorized.GET("api/rclone/commands", func(c *gin.Context) { rcloneHandler.RcloneCommandOptions(c) }) + authorized.GET("api/rclone/command-flags", func(c *gin.Context) { rcloneHandler.RcloneCommandFlags(c) }) + authorized.GET("api/rclone/command/:id/usage", func(c *gin.Context) { rcloneHandler.RcloneCommandUsage(c) }) + authorized.GET("/jobs", h.HandleJobs) authorized.GET("/jobs/new", h.HandleNewJob) authorized.GET("/jobs/:id", h.HandleEditJob)