mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-09-20 13:30:51 +02:00
feat: Integrate rclone command functionality into configuration management
- Added support for selecting rclone commands and their associated flags in the configuration form. - Implemented backend logic to handle rclone command retrieval and flag management. - Enhanced the database schema to include rclone command and flag fields in the transfer configuration. - Created new migration scripts to add rclone-related tables and fields. - Updated the scheduler to execute transfers using the selected rclone command and flags. - Developed new handlers for rendering rclone command options and flags in the web interface. - Improved error handling and logging for rclone command execution and configuration updates.
This commit is contained in:
StarFleetCPTN
+374
-3
@@ -120,9 +120,12 @@ type TransferConfig struct {
|
||||
UseBuiltinAuthDest *bool `form:"use_builtin_auth_dest"` // For Google and other OAuth services
|
||||
GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed
|
||||
// General fields
|
||||
ArchivePath string `form:"archive_path"`
|
||||
ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"`
|
||||
RcloneFlags string `form:"rclone_flags"`
|
||||
ArchivePath string `form:"archive_path"`
|
||||
ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"`
|
||||
RcloneFlags string `form:"rclone_flags"`
|
||||
// Rclone command fields
|
||||
CommandID uint `gorm:"default:1" form:"command_id"` // Default to 'copy' command ID (1)
|
||||
CommandFlags string `form:"command_flags"` // JSON string of selected flags
|
||||
DeleteAfterTransfer *bool `gorm:"default:false" form:"delete_after_transfer"`
|
||||
SkipProcessedFiles *bool `gorm:"default:true" form:"skip_processed_files"`
|
||||
MaxConcurrentTransfers int `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers
|
||||
@@ -250,6 +253,31 @@ type DB struct {
|
||||
*gorm.DB
|
||||
}
|
||||
|
||||
// RcloneCommand represents a command available in rclone
|
||||
type RcloneCommand struct {
|
||||
ID uint `gorm:"primarykey"`
|
||||
Name string `gorm:"not null;uniqueIndex"`
|
||||
Description string `gorm:"not null"`
|
||||
Category string `gorm:"not null;index"`
|
||||
IsAdvanced bool `gorm:"not null;default:false"`
|
||||
Flags []RcloneCommandFlag `gorm:"foreignKey:CommandID;constraint:OnDelete:CASCADE"`
|
||||
CreatedAt time.Time `gorm:"not null"`
|
||||
}
|
||||
|
||||
// RcloneCommandFlag represents a flag that can be used with an rclone command
|
||||
type RcloneCommandFlag struct {
|
||||
ID uint `gorm:"primarykey"`
|
||||
CommandID uint `gorm:"not null;index"`
|
||||
Command RcloneCommand `gorm:"foreignKey:CommandID"`
|
||||
Name string `gorm:"not null;index"`
|
||||
ShortName string
|
||||
Description string `gorm:"not null"`
|
||||
DataType string `gorm:"not null"` // string, int, bool, etc.
|
||||
IsRequired bool `gorm:"not null;default:false"`
|
||||
DefaultValue string
|
||||
CreatedAt time.Time `gorm:"not null"`
|
||||
}
|
||||
|
||||
func Initialize(dbPath string) (*DB, error) {
|
||||
// Create directory if it doesn't exist
|
||||
dir := filepath.Dir(dbPath)
|
||||
@@ -1707,3 +1735,346 @@ func (u *User) CheckPassword(password string) bool {
|
||||
err := bcrypt.CompareHashAndPassword([]byte(u.PasswordHash), []byte(password))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// GetRcloneCommands returns all rclone commands
|
||||
func (db *DB) GetRcloneCommands() ([]RcloneCommand, error) {
|
||||
var commands []RcloneCommand
|
||||
err := db.Find(&commands).Error
|
||||
return commands, err
|
||||
}
|
||||
|
||||
// GetRcloneCommand returns a specific rclone command by ID
|
||||
func (db *DB) GetRcloneCommand(id uint) (*RcloneCommand, error) {
|
||||
var command RcloneCommand
|
||||
err := db.First(&command, id).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &command, nil
|
||||
}
|
||||
|
||||
// GetRcloneCommandByName returns a specific rclone command by name
|
||||
func (db *DB) GetRcloneCommandByName(name string) (*RcloneCommand, error) {
|
||||
var command RcloneCommand
|
||||
err := db.Where("name = ?", name).First(&command).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &command, nil
|
||||
}
|
||||
|
||||
// GetRcloneCommandsInCategory returns all commands in a specific category
|
||||
func (db *DB) GetRcloneCommandsInCategory(category string) ([]RcloneCommand, error) {
|
||||
var commands []RcloneCommand
|
||||
err := db.Where("category = ?", category).Find(&commands).Error
|
||||
return commands, err
|
||||
}
|
||||
|
||||
// GetRcloneCommandFlag returns a specific flag by ID
|
||||
func (db *DB) GetRcloneCommandFlag(id uint) (*RcloneCommandFlag, error) {
|
||||
var flag RcloneCommandFlag
|
||||
err := db.First(&flag, id).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &flag, nil
|
||||
}
|
||||
|
||||
// GetRcloneCommandFlagByName returns a specific flag by name for a command
|
||||
func (db *DB) GetRcloneCommandFlagByName(commandID uint, name string) (*RcloneCommandFlag, error) {
|
||||
var flag RcloneCommandFlag
|
||||
err := db.Where("command_id = ? AND name = ?", commandID, name).First(&flag).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &flag, nil
|
||||
}
|
||||
|
||||
// GetRcloneCommandFlags returns all flags for a specific command
|
||||
func (db *DB) GetRcloneCommandFlags(commandID uint) ([]RcloneCommandFlag, error) {
|
||||
var flags []RcloneCommandFlag
|
||||
err := db.Where("command_id = ?", commandID).Find(&flags).Error
|
||||
return flags, err
|
||||
}
|
||||
|
||||
// GetRcloneCommandWithFlags returns a command with all its flags
|
||||
func (db *DB) GetRcloneCommandWithFlags(commandID uint) (*RcloneCommand, error) {
|
||||
var command RcloneCommand
|
||||
err := db.Preload("Flags").First(&command, commandID).Error
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &command, nil
|
||||
}
|
||||
|
||||
// BuildRcloneCommand builds an rclone command string with the specified command and flags
|
||||
func (db *DB) BuildRcloneCommand(commandName string, flags map[string]string) (string, error) {
|
||||
// Get the command details
|
||||
command, err := db.GetRcloneCommandByName(commandName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("command not found: %s", commandName)
|
||||
}
|
||||
|
||||
// Start building the command string
|
||||
cmdStr := "rclone " + command.Name
|
||||
|
||||
// Get all flags for this command
|
||||
allFlags, err := db.GetRcloneCommandFlags(command.ID)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to get flags for command: %v", err)
|
||||
}
|
||||
|
||||
// Create a map of flag details for easy lookup
|
||||
flagDetails := make(map[string]RcloneCommandFlag)
|
||||
for _, f := range allFlags {
|
||||
flagDetails[f.Name] = f
|
||||
}
|
||||
|
||||
// Add the flags to the command
|
||||
for name, value := range flags {
|
||||
// Check if the flag exists for this command
|
||||
flag, exists := flagDetails[name]
|
||||
if !exists {
|
||||
return "", fmt.Errorf("invalid flag for command %s: %s", commandName, name)
|
||||
}
|
||||
|
||||
// Handle different flag types
|
||||
switch flag.DataType {
|
||||
case "bool":
|
||||
if value == "true" {
|
||||
cmdStr += " " + name
|
||||
}
|
||||
default:
|
||||
cmdStr += " " + name + " " + value
|
||||
}
|
||||
}
|
||||
|
||||
return cmdStr, nil
|
||||
}
|
||||
|
||||
// ValidateRcloneFlags validates if the provided flags are valid for the command
|
||||
func (db *DB) ValidateRcloneFlags(commandName string, flags map[string]string) (bool, map[string]string) {
|
||||
// Initialize errors map
|
||||
errors := make(map[string]string)
|
||||
|
||||
// Get the command details
|
||||
command, err := db.GetRcloneCommandByName(commandName)
|
||||
if err != nil {
|
||||
errors["command"] = "Command not found: " + commandName
|
||||
return false, errors
|
||||
}
|
||||
|
||||
// Get all flags for this command
|
||||
allFlags, err := db.GetRcloneCommandFlags(command.ID)
|
||||
if err != nil {
|
||||
errors["command"] = "Failed to get flags for command"
|
||||
return false, errors
|
||||
}
|
||||
|
||||
// Create a map of flag details for easy lookup
|
||||
flagDetails := make(map[string]RcloneCommandFlag)
|
||||
for _, f := range allFlags {
|
||||
flagDetails[f.Name] = f
|
||||
}
|
||||
|
||||
// Check each provided flag
|
||||
for name, value := range flags {
|
||||
// Check if the flag exists for this command
|
||||
flag, exists := flagDetails[name]
|
||||
if !exists {
|
||||
errors[name] = "Invalid flag for command " + commandName
|
||||
continue
|
||||
}
|
||||
|
||||
// Validate the flag value based on data type
|
||||
switch flag.DataType {
|
||||
case "int":
|
||||
if _, err := strconv.Atoi(value); err != nil {
|
||||
errors[name] = "Value must be an integer"
|
||||
}
|
||||
case "float":
|
||||
if _, err := strconv.ParseFloat(value, 64); err != nil {
|
||||
errors[name] = "Value must be a number"
|
||||
}
|
||||
case "bool":
|
||||
if value != "true" && value != "false" {
|
||||
errors[name] = "Value must be true or false"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check for required flags
|
||||
for _, flag := range allFlags {
|
||||
if flag.IsRequired {
|
||||
if _, provided := flags[flag.Name]; !provided {
|
||||
errors[flag.Name] = "This flag is required"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return len(errors) == 0, errors
|
||||
}
|
||||
|
||||
// GetRcloneCategories returns all unique categories of rclone commands
|
||||
func (db *DB) GetRcloneCategories() ([]string, error) {
|
||||
var categories []string
|
||||
err := db.Model(&RcloneCommand{}).Distinct("category").Pluck("category", &categories).Error
|
||||
return categories, err
|
||||
}
|
||||
|
||||
// GetRcloneCommandsByAdvanced returns commands filtered by their advanced status
|
||||
func (db *DB) GetRcloneCommandsByAdvanced(isAdvanced bool) ([]RcloneCommand, error) {
|
||||
var commands []RcloneCommand
|
||||
err := db.Where("is_advanced = ?", isAdvanced).Find(&commands).Error
|
||||
return commands, err
|
||||
}
|
||||
|
||||
// SearchRcloneCommands searches for commands by name or description
|
||||
func (db *DB) SearchRcloneCommands(query string) ([]RcloneCommand, error) {
|
||||
var commands []RcloneCommand
|
||||
searchQuery := "%" + query + "%"
|
||||
err := db.Where("name LIKE ? OR description LIKE ?", searchQuery, searchQuery).Find(&commands).Error
|
||||
return commands, err
|
||||
}
|
||||
|
||||
// GetRcloneFlagUsage returns a human-readable usage example for a flag
|
||||
func (flag *RcloneCommandFlag) GetUsageExample() string {
|
||||
switch flag.DataType {
|
||||
case "bool":
|
||||
return flag.Name
|
||||
case "int":
|
||||
return fmt.Sprintf("%s=<number>", flag.Name)
|
||||
case "float":
|
||||
return fmt.Sprintf("%s=<decimal>", flag.Name)
|
||||
case "string":
|
||||
return fmt.Sprintf("%s=<text>", flag.Name)
|
||||
default:
|
||||
return fmt.Sprintf("%s=<value>", flag.Name)
|
||||
}
|
||||
}
|
||||
|
||||
// GetRcloneCommandUsage returns a basic usage example for a command with its required flags
|
||||
func (db *DB) GetRcloneCommandUsage(commandID uint) (string, error) {
|
||||
command, err := db.GetRcloneCommandWithFlags(commandID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
usage := fmt.Sprintf("rclone %s [flags] <source> <dest>", command.Name)
|
||||
|
||||
// Add basic usage examples for required flags
|
||||
requiredFlags := []string{}
|
||||
for _, flag := range command.Flags {
|
||||
if flag.IsRequired {
|
||||
requiredFlags = append(requiredFlags, flag.GetUsageExample())
|
||||
}
|
||||
}
|
||||
|
||||
if len(requiredFlags) > 0 {
|
||||
usage += "\n\nRequired flags:\n " + strings.Join(requiredFlags, "\n ")
|
||||
}
|
||||
|
||||
return usage, nil
|
||||
}
|
||||
|
||||
// ParseRcloneFlags parses a string of rclone flags into a map
|
||||
func ParseRcloneFlags(flagsStr string) map[string]string {
|
||||
result := make(map[string]string)
|
||||
if flagsStr == "" {
|
||||
return result
|
||||
}
|
||||
|
||||
// Split the flags string by spaces
|
||||
parts := strings.Fields(flagsStr)
|
||||
|
||||
for i := 0; i < len(parts); i++ {
|
||||
part := parts[i]
|
||||
|
||||
// Check if it's a flag (starts with --)
|
||||
if strings.HasPrefix(part, "--") {
|
||||
// Remove the -- prefix
|
||||
flagName := part
|
||||
|
||||
// Check if the flag has a value
|
||||
if i+1 < len(parts) && !strings.HasPrefix(parts[i+1], "--") {
|
||||
// Next part is a value
|
||||
result[flagName] = parts[i+1]
|
||||
i++ // Skip the value in the next iteration
|
||||
} else {
|
||||
// Flag without value, treat as boolean
|
||||
result[flagName] = "true"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// RenderRcloneCommandHelp generates a help text for a command with its flags
|
||||
func (db *DB) RenderRcloneCommandHelp(commandID uint) (string, error) {
|
||||
command, err := db.GetRcloneCommandWithFlags(commandID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Build the help text
|
||||
help := fmt.Sprintf("COMMAND: %s\n", command.Name)
|
||||
help += fmt.Sprintf("DESCRIPTION: %s\n\n", command.Description)
|
||||
help += "FLAGS:\n"
|
||||
|
||||
// Group flags by required status
|
||||
var requiredFlags, optionalFlags []RcloneCommandFlag
|
||||
for _, flag := range command.Flags {
|
||||
if flag.IsRequired {
|
||||
requiredFlags = append(requiredFlags, flag)
|
||||
} else {
|
||||
optionalFlags = append(optionalFlags, flag)
|
||||
}
|
||||
}
|
||||
|
||||
// Add required flags
|
||||
if len(requiredFlags) > 0 {
|
||||
help += " Required:\n"
|
||||
for _, flag := range requiredFlags {
|
||||
shortName := ""
|
||||
if flag.ShortName != "" {
|
||||
shortName = fmt.Sprintf(" (-%s)", flag.ShortName)
|
||||
}
|
||||
help += fmt.Sprintf(" %s%s - %s\n", flag.Name, shortName, flag.Description)
|
||||
if flag.DataType != "bool" && flag.DefaultValue != "" {
|
||||
help += fmt.Sprintf(" Default: %s\n", flag.DefaultValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add optional flags
|
||||
if len(optionalFlags) > 0 {
|
||||
help += "\n Optional:\n"
|
||||
for _, flag := range optionalFlags {
|
||||
shortName := ""
|
||||
if flag.ShortName != "" {
|
||||
shortName = fmt.Sprintf(" (-%s)", flag.ShortName)
|
||||
}
|
||||
help += fmt.Sprintf(" %s%s - %s\n", flag.Name, shortName, flag.Description)
|
||||
if flag.DataType != "bool" && flag.DefaultValue != "" {
|
||||
help += fmt.Sprintf(" Default: %s\n", flag.DefaultValue)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return help, nil
|
||||
}
|
||||
|
||||
func (db *DB) GetRcloneCommandFlagsMap(commandID uint) (map[uint]RcloneCommandFlag, error) {
|
||||
flags, err := db.GetRcloneCommandFlags(commandID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
flagsMap := make(map[uint]RcloneCommandFlag)
|
||||
for _, flag := range flags {
|
||||
flagsMap[flag.ID] = flag
|
||||
}
|
||||
|
||||
return flagsMap, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,349 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AddRcloneTables adds tables for rclone commands and their flags
|
||||
func AddRcloneTables() *gormigrate.Migration {
|
||||
return &gormigrate.Migration{
|
||||
ID: "009_add_rclone_tables",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// Check if any tables exist (indicating an existing database)
|
||||
var count int64
|
||||
if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil {
|
||||
return fmt.Errorf("failed to check for existing tables: %v", err)
|
||||
}
|
||||
|
||||
// If tables exist, create a backup
|
||||
if count > 0 {
|
||||
// Get the database path
|
||||
sqlDB, err := tx.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get underlying database: %v", err)
|
||||
}
|
||||
|
||||
var seq int
|
||||
var name, dbPath string
|
||||
if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil {
|
||||
return fmt.Errorf("failed to get database path: %v", err)
|
||||
}
|
||||
|
||||
// Get backup directory from environment variable or use default
|
||||
backupDir := os.Getenv("BACKUP_DIR")
|
||||
if backupDir == "" {
|
||||
backupDir = "/app/backups" // Default Docker path
|
||||
// Check if we're not in Docker
|
||||
if _, err := os.Stat(backupDir); os.IsNotExist(err) {
|
||||
backupDir = "backups" // Fallback to local directory
|
||||
}
|
||||
}
|
||||
|
||||
// Create backup directory if it doesn't exist
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create backup directory: %v", err)
|
||||
}
|
||||
|
||||
// Create backup file with timestamp in the backup directory
|
||||
dbFileName := filepath.Base(dbPath)
|
||||
backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405"))
|
||||
backupFile := filepath.Join(backupDir, backupFileName)
|
||||
|
||||
// Read original database
|
||||
data, err := os.ReadFile(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read database for backup: %v", err)
|
||||
}
|
||||
|
||||
// Write backup
|
||||
if err := os.WriteFile(backupFile, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to create database backup: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Created database backup at: %s\n", backupFile)
|
||||
}
|
||||
|
||||
// Create the rclone_commands table
|
||||
err := tx.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS rclone_commands (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
is_advanced BOOLEAN NOT NULL DEFAULT 0,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create an index on name for faster lookups
|
||||
err = tx.Exec(`
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_rclone_commands_name ON rclone_commands(name)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create an index on category for faster filtering
|
||||
err = tx.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_rclone_commands_category ON rclone_commands(category)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create the rclone_command_flags table
|
||||
err = tx.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS rclone_command_flags (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
command_id INTEGER NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
short_name TEXT,
|
||||
description TEXT NOT NULL,
|
||||
data_type TEXT NOT NULL,
|
||||
is_required BOOLEAN NOT NULL DEFAULT 0,
|
||||
default_value TEXT,
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (command_id) REFERENCES rclone_commands(id) ON DELETE CASCADE
|
||||
)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create index on command_id for faster lookups
|
||||
err = tx.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_command_id ON rclone_command_flags(command_id)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Create index on flag name for faster searches
|
||||
err = tx.Exec(`
|
||||
CREATE INDEX IF NOT EXISTS idx_rclone_command_flags_name ON rclone_command_flags(name)
|
||||
`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Populate the tables with default rclone commands and flags
|
||||
return populateRcloneTablesWithDefaults(tx)
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// Drop the flags table first due to foreign key constraints
|
||||
err := tx.Exec(`DROP TABLE IF EXISTS rclone_command_flags`).Error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Then drop the commands table
|
||||
return tx.Exec(`DROP TABLE IF EXISTS rclone_commands`).Error
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to populate rclone tables with default data
|
||||
func populateRcloneTablesWithDefaults(tx *gorm.DB) error {
|
||||
// Define base commands
|
||||
commands := []struct {
|
||||
Name string
|
||||
Description string
|
||||
Category string
|
||||
IsAdvanced bool
|
||||
}{
|
||||
// Core commands
|
||||
{"copy", "Copy files from source to dest, skipping already copied", "sync", false},
|
||||
{"sync", "Make source and dest identical, modifying destination only", "sync", false},
|
||||
{"bisync", "Bidirectional synchronization between two paths", "sync", false},
|
||||
{"move", "Move files from source to dest", "sync", false},
|
||||
{"delete", "Remove the contents of path", "sync", false},
|
||||
{"purge", "Remove the path and all of its contents", "sync", false},
|
||||
{"mkdir", "Make the path if it doesn't already exist", "sync", false},
|
||||
{"rmdir", "Remove the path", "sync", false},
|
||||
{"rmdirs", "Remove any empty directories under the path", "sync", false},
|
||||
{"check", "Check if the files in the source and destination match", "sync", false},
|
||||
{"ls", "List all the objects in the path with size and path", "listing", false},
|
||||
{"lsd", "List all directories/containers/buckets in the path", "listing", false},
|
||||
{"lsl", "List all the objects in the path with size, modification time and path", "listing", false},
|
||||
{"lsf", "List the objects in the path (obey formatting parameters)", "listing", false},
|
||||
{"lsjson", "List the objects in the path in JSON format", "listing", false},
|
||||
{"md5sum", "Produce an md5sum file for all the objects in the path", "hash", false},
|
||||
{"sha1sum", "Produce a sha1sum file for all the objects in the path", "hash", false},
|
||||
{"size", "Return the total size and number of objects in remote:path", "info", false},
|
||||
{"version", "Show the version number", "info", false},
|
||||
{"cleanup", "Clean up the remote if possible", "maintenance", false},
|
||||
{"dedupe", "Interactively find duplicate files and delete/rename them", "maintenance", false},
|
||||
{"copyto", "Copy files from source to dest, skipping already copied", "sync", false},
|
||||
{"moveto", "Move file or directory from source to dest", "sync", false},
|
||||
{"listremotes", "List all the remotes in the config file", "config", false},
|
||||
{"obscure", "Obscure password for use in the rclone.conf", "config", false},
|
||||
{"cryptcheck", "Check the integrity of an encrypted remote", "crypto", false},
|
||||
}
|
||||
|
||||
// Insert commands
|
||||
for _, cmd := range commands {
|
||||
err := tx.Exec(`
|
||||
INSERT INTO rclone_commands
|
||||
(name, description, category, is_advanced, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)
|
||||
`, cmd.Name, cmd.Description, cmd.Category, cmd.IsAdvanced, time.Now()).Error
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert command %s: %v", cmd.Name, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Define all flags
|
||||
allFlags := []struct {
|
||||
CommandName string
|
||||
Name string
|
||||
ShortName string
|
||||
Description string
|
||||
DataType string
|
||||
IsRequired bool
|
||||
DefaultValue string
|
||||
}{
|
||||
// Global flags - apply to most commands
|
||||
{"global", "transfers", "n", "Number of file transfers to run in parallel", "int", false, "4"},
|
||||
{"global", "checkers", "", "Number of checkers to run in parallel", "int", false, "8"},
|
||||
{"global", "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR", "string", false, "NOTICE"},
|
||||
{"global", "stats", "", "Interval between logging stats, e.g 500ms, 60s, 5m", "string", false, "1m0s"},
|
||||
{"global", "stats-file-name-length", "", "Max file name length in stats", "int", false, "45"},
|
||||
{"global", "stats-one-line", "", "Make the stats fit on one line", "bool", false, "false"},
|
||||
{"global", "progress", "p", "Show progress during transfer", "bool", false, "false"},
|
||||
{"global", "verbose", "v", "Show verbose output", "bool", false, "false"},
|
||||
{"global", "quiet", "q", "Print as little stuff as possible", "bool", false, "false"},
|
||||
{"global", "retries", "", "Retry operations this many times if they fail", "int", false, "3"},
|
||||
{"global", "retries-sleep", "", "Interval between retrying operations if they fail", "string", false, "0s"},
|
||||
{"global", "timeout", "", "IO idle timeout", "string", false, "5m0s"},
|
||||
{"global", "tpslimit", "", "Limit HTTP transactions per second", "float", false, "0"},
|
||||
{"global", "tpslimit-burst", "", "Max burst of transactions for --tpslimit", "int", false, "1"},
|
||||
{"global", "size-only", "", "Skip based on size only, not mod-time or checksum", "bool", false, "false"},
|
||||
{"global", "ignore-checksum", "", "Skip post copy check of checksums", "bool", false, "false"},
|
||||
{"global", "ignore-existing", "", "Skip all files that exist on destination", "bool", false, "false"},
|
||||
{"global", "ignore-size", "", "Skip size checks to calculate if file changed", "bool", false, "false"},
|
||||
{"global", "ignore-case-sync", "", "Ignore case when synchronizing", "bool", false, "false"},
|
||||
{"global", "no-update-modtime", "", "Don't update destination mod-time if files identical", "bool", false, "false"},
|
||||
{"global", "no-check-certificate", "", "Do not verify server SSL certificates", "bool", false, "false"},
|
||||
{"global", "ask-password", "", "Allow prompt for password for encrypted config", "bool", false, "false"},
|
||||
{"global", "dump", "", "List of items to dump from: headers,bodies,requests,responses,auth,filters,goroutines,openfiles", "string", false, ""},
|
||||
{"global", "metadata", "M", "Preserve metadata when copying objects", "bool", false, "false"},
|
||||
{"global", "metadata-set", "", "Add metadata key=value when uploading", "string", false, ""},
|
||||
|
||||
// copy command specific flags
|
||||
{"copy", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
{"copy", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
|
||||
{"copy", "cutoff-mode", "", "Mode to stop transfers when reaching the cutoff threshold", "string", false, "hard"},
|
||||
{"copy", "max-transfer", "", "Maximum size of data to transfer", "string", false, "off"},
|
||||
{"copy", "max-backlog", "", "Maximum size of upload or download backlog", "int", false, "10000"},
|
||||
{"copy", "track-renames", "", "Track file renames during sync", "bool", false, "false"},
|
||||
{"copy", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"},
|
||||
|
||||
// sync command specific flags
|
||||
{"sync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
{"sync", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
|
||||
{"sync", "backup-dir", "", "Make backups into this directory", "string", false, ""},
|
||||
{"sync", "suffix", "", "Suffix to add to changed files", "string", false, ""},
|
||||
{"sync", "delete-before", "", "Delete before transferring", "bool", false, "false"},
|
||||
{"sync", "delete-during", "", "Delete during transferring", "bool", false, "false"},
|
||||
{"sync", "delete-after", "", "Delete after transferring", "bool", false, "false"},
|
||||
{"sync", "track-renames", "", "Track file renames during sync", "bool", false, "false"},
|
||||
{"sync", "track-renames-strategy", "", "Strategies to detect renames (hash|modtime|leaf)", "string", false, "hash"},
|
||||
|
||||
// bisync command specific flags
|
||||
{"bisync", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
{"bisync", "resync", "", "Performs the resync run", "bool", false, "false"},
|
||||
{"bisync", "check-access", "", "Ensure destination has write access", "bool", false, "true"},
|
||||
{"bisync", "conflict-resolve", "", "Automatically resolve conflicts (newer|larger|older|smaller)", "string", false, ""},
|
||||
{"bisync", "max-delete", "", "Safety check on maximum files to delete", "int", false, "10"},
|
||||
|
||||
// move command specific flags
|
||||
{"move", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
{"move", "create-empty-src-dirs", "", "Create empty source dirs on destination", "bool", false, "false"},
|
||||
{"move", "delete-empty-src-dirs", "", "Delete empty source dirs after move", "bool", false, "false"},
|
||||
|
||||
// ls/listing related flags
|
||||
{"ls", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
|
||||
{"ls", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
|
||||
{"ls", "format", "", "Format for the output", "string", false, ""},
|
||||
{"ls", "absolute", "", "Put a leading / in front of path names", "bool", false, "false"},
|
||||
|
||||
{"lsl", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
|
||||
{"lsl", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
|
||||
|
||||
{"lsd", "max-depth", "", "Maximum depth to show in the listing", "int", false, ""},
|
||||
{"lsd", "dir-sort", "", "Directory sorting (alphabetical|size|time)", "string", false, "alphabetical"},
|
||||
|
||||
{"lsf", "format", "", "Format for the output", "string", false, ""},
|
||||
{"lsf", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
|
||||
{"lsf", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
|
||||
|
||||
{"lsjson", "recursive", "R", "Recurse into the listing", "bool", false, "false"},
|
||||
{"lsjson", "max-depth", "", "Maximum depth to recursively list", "int", false, ""},
|
||||
{"lsjson", "files-only", "", "Show only files, not directories", "bool", false, "false"},
|
||||
{"lsjson", "encrypted", "", "Show the encrypted names", "bool", false, "false"},
|
||||
{"lsjson", "stat", "", "Show status of objects", "bool", false, "false"},
|
||||
|
||||
// Other specialized command flags
|
||||
{"cryptcheck", "one-way", "", "Check one way only, source files must exist on destination", "bool", false, "false"},
|
||||
|
||||
{"cleanup", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
|
||||
{"dedupe", "dry-run", "", "Do a trial run with no permanent changes", "bool", false, "false"},
|
||||
{"dedupe", "mode", "", "Dedupe mode interactive|skip|first|newest|oldest|largest|smallest|rename", "string", false, "interactive"},
|
||||
}
|
||||
|
||||
// Insert flags with associated command IDs
|
||||
for _, flag := range allFlags {
|
||||
// For global flags, add them to all commands
|
||||
if flag.CommandName == "global" {
|
||||
// Get all command IDs
|
||||
var commandIDs []int64
|
||||
err := tx.Raw(`SELECT id FROM rclone_commands`).Scan(&commandIDs).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get all command IDs: %v", err)
|
||||
}
|
||||
|
||||
// Add global flag to each command
|
||||
for _, cmdID := range commandIDs {
|
||||
err = tx.Exec(`
|
||||
INSERT INTO rclone_command_flags
|
||||
(command_id, name, short_name, description, data_type, is_required, default_value, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, cmdID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert global flag %s for command ID %d: %v", flag.Name, cmdID, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Add command-specific flag
|
||||
var commandID int64
|
||||
err := tx.Raw(`SELECT id FROM rclone_commands WHERE name = ?`, flag.CommandName).Scan(&commandID).Error
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to find command %s: %v", flag.CommandName, err)
|
||||
}
|
||||
|
||||
err = tx.Exec(`
|
||||
INSERT INTO rclone_command_flags
|
||||
(command_id, name, short_name, description, data_type, is_required, default_value, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`, commandID, flag.Name, flag.ShortName, flag.Description, flag.DataType, flag.IsRequired, flag.DefaultValue, time.Now()).Error
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to insert flag %s for command %s: %v", flag.Name, flag.CommandName, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AddRcloneCommandToConfig adds rclone command fields to the transfer_configs table
|
||||
func AddRcloneCommandToConfig() *gormigrate.Migration {
|
||||
return &gormigrate.Migration{
|
||||
ID: "010_add_rclone_command_to_config",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// Check if any tables exist (indicating an existing database)
|
||||
var count int64
|
||||
if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil {
|
||||
return fmt.Errorf("failed to check for existing tables: %v", err)
|
||||
}
|
||||
|
||||
// If tables exist, create a backup
|
||||
if count > 0 {
|
||||
// Get the database path
|
||||
sqlDB, err := tx.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get underlying database: %v", err)
|
||||
}
|
||||
|
||||
var seq int
|
||||
var name, dbPath string
|
||||
if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil {
|
||||
return fmt.Errorf("failed to get database path: %v", err)
|
||||
}
|
||||
|
||||
// Get backup directory from environment variable or use default
|
||||
backupDir := os.Getenv("BACKUP_DIR")
|
||||
if backupDir == "" {
|
||||
backupDir = "/app/backups" // Default Docker path
|
||||
// Check if we're not in Docker
|
||||
if _, err := os.Stat(backupDir); os.IsNotExist(err) {
|
||||
backupDir = "backups" // Fallback to local directory
|
||||
}
|
||||
}
|
||||
|
||||
// Create backup directory if it doesn't exist
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create backup directory: %v", err)
|
||||
}
|
||||
|
||||
// Create backup file with timestamp in the backup directory
|
||||
dbFileName := filepath.Base(dbPath)
|
||||
backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405"))
|
||||
backupFile := filepath.Join(backupDir, backupFileName)
|
||||
|
||||
// Read original database
|
||||
data, err := os.ReadFile(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read database for backup: %v", err)
|
||||
}
|
||||
|
||||
// Write backup
|
||||
if err := os.WriteFile(backupFile, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to create database backup: %v", err)
|
||||
}
|
||||
|
||||
fmt.Printf("Created database backup at: %s\n", backupFile)
|
||||
}
|
||||
|
||||
// Add the command_id and command_flags columns to the transfer_configs table
|
||||
if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_id INTEGER DEFAULT NULL REFERENCES rclone_commands(id)`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Exec(`ALTER TABLE transfer_configs ADD COLUMN command_flags TEXT DEFAULT NULL`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// Remove the columns in reverse order
|
||||
if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_flags`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Exec(`ALTER TABLE transfer_configs DROP COLUMN command_id`).Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,8 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate {
|
||||
AddTimestampsToJobHistories(), // 006
|
||||
AddNotificationServices(), // 007
|
||||
AddUserNotifications(), // 008
|
||||
AddRcloneTables(), // 009
|
||||
AddRcloneCommandToConfig(), // 010
|
||||
)
|
||||
|
||||
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
|
||||
|
||||
Reference in New Issue
Block a user