feat: Add support for skipping processed files in transfer configs

- Introduce new `skip_processed_files` option for transfer configurations
- Add database migration to support the new column
- Update config form and UI to include skip processed files toggle
- Enhance scheduler to respect skip processed files setting
- Modify file processing logic to optionally skip previously processed files
- Add environment variable support for skip processed files configuration

internal/config/config.go

@@ -1,31 +1,34 @@
 package config
 
 import (
-	"encoding/json"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/joho/godotenv"
 )
 
 type Config struct {
-	ServerAddress string `json:"server_address"`
-	DataDir       string `json:"data_dir"`
-	BackupDir     string `json:"backup_dir"`
-	JWTSecret     string `json:"jwt_secret"`
+	ServerAddress string      `json:"server_address"`
+	DataDir       string      `json:"data_dir"`
+	BackupDir     string      `json:"backup_dir"`
+	JWTSecret     string      `json:"jwt_secret"`
 	Email         EmailConfig `json:"email"`
-	BaseURL       string `json:"base_url"` // Base URL for generating links in emails
+	BaseURL       string      `json:"base_url"` // Base URL for generating links in emails
 }
 
 type EmailConfig struct {
-	Enabled    bool   `json:"enabled"`
-	Host       string `json:"host"`
-	Port       int    `json:"port"`
-	Username   string `json:"username"`
-	Password   string `json:"password"`
-	FromEmail  string `json:"from_email"`
-	FromName   string `json:"from_name"`
-	ReplyTo    string `json:"reply_to,omitempty"`
-	EnableTLS  bool   `json:"enable_tls"`
-	RequireAuth bool  `json:"require_auth"`
+	Enabled     bool   `json:"enabled"`
+	Host        string `json:"host"`
+	Port        int    `json:"port"`
+	Username    string `json:"username"`
+	Password    string `json:"password"`
+	FromEmail   string `json:"from_email"`
+	FromName    string `json:"from_name"`
+	ReplyTo     string `json:"reply_to,omitempty"`
+	EnableTLS   bool   `json:"enable_tls"`
+	RequireAuth bool   `json:"require_auth"`
 }
 
 func Load() (*Config, error) {
@@ -37,48 +40,106 @@ func Load() (*Config, error) {
 		JWTSecret:     "change_this_to_a_secure_random_string",
 		BaseURL:       "http://localhost:8080",
 		Email: EmailConfig{
-			Enabled:    false,
-			Host:       "smtp.example.com",
-			Port:       587,
-			Username:   "user@example.com",
-			Password:   "your-password",
-			FromEmail:  "gomft@example.com",
-			FromName:   "GoMFT",
-			EnableTLS:  true,
+			Enabled:     false,
+			Host:        "smtp.example.com",
+			Port:        587,
+			Username:    "user@example.com",
+			Password:    "your-password",
+			FromEmail:   "gomft@example.com",
+			FromName:    "GoMFT",
+			EnableTLS:   true,
 			RequireAuth: true,
 		},
 	}
 
-	// Check if config file exists
-	configPath := filepath.Join(cfg.DataDir, "config.json")
-	if _, err := os.Stat(configPath); err == nil {
-		// Read configuration file
-		data, err := os.ReadFile(configPath)
-		if err != nil {
-			return nil, err
-		}
-
-		// Parse configuration
-		if err := json.Unmarshal(data, cfg); err != nil {
-			return nil, err
-		}
-	} else if !os.IsNotExist(err) {
-		return nil, err
-	}
-
 	// Ensure data directory exists
 	if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
 		return nil, err
 	}
 
-	// Save configuration if it doesn't exist
-	if _, err := os.Stat(configPath); os.IsNotExist(err) {
-		data, err := json.MarshalIndent(cfg, "", "  ")
-		if err != nil {
+	// First try to load .env from the root directory
+	envPath := ".env"
+	if _, err := os.Stat(envPath); err == nil {
+		// Load .env file
+		if err := godotenv.Load(envPath); err != nil {
 			return nil, err
 		}
 
-		if err := os.WriteFile(configPath, data, 0644); err != nil {
+		// Override configuration with environment variables
+		if serverAddr := os.Getenv("SERVER_ADDRESS"); serverAddr != "" {
+			cfg.ServerAddress = serverAddr
+		}
+		if dataDir := os.Getenv("DATA_DIR"); dataDir != "" {
+			cfg.DataDir = dataDir
+		}
+		if backupDir := os.Getenv("BACKUP_DIR"); backupDir != "" {
+			cfg.BackupDir = backupDir
+		}
+		if jwtSecret := os.Getenv("JWT_SECRET"); jwtSecret != "" {
+			cfg.JWTSecret = jwtSecret
+		}
+		if baseURL := os.Getenv("BASE_URL"); baseURL != "" {
+			cfg.BaseURL = baseURL
+		}
+
+		// Email configuration
+		if emailEnabled := os.Getenv("EMAIL_ENABLED"); emailEnabled != "" {
+			cfg.Email.Enabled = strings.ToLower(emailEnabled) == "true"
+		}
+		if emailHost := os.Getenv("EMAIL_HOST"); emailHost != "" {
+			cfg.Email.Host = emailHost
+		}
+		if emailPort := os.Getenv("EMAIL_PORT"); emailPort != "" {
+			if port, err := strconv.Atoi(emailPort); err == nil {
+				cfg.Email.Port = port
+			}
+		}
+		if emailUsername := os.Getenv("EMAIL_USERNAME"); emailUsername != "" {
+			cfg.Email.Username = emailUsername
+		}
+		if emailPassword := os.Getenv("EMAIL_PASSWORD"); emailPassword != "" {
+			cfg.Email.Password = emailPassword
+		}
+		if emailFromEmail := os.Getenv("EMAIL_FROM_EMAIL"); emailFromEmail != "" {
+			cfg.Email.FromEmail = emailFromEmail
+		}
+		if emailFromName := os.Getenv("EMAIL_FROM_NAME"); emailFromName != "" {
+			cfg.Email.FromName = emailFromName
+		}
+		if emailReplyTo := os.Getenv("EMAIL_REPLY_TO"); emailReplyTo != "" {
+			cfg.Email.ReplyTo = emailReplyTo
+		}
+		if emailEnableTLS := os.Getenv("EMAIL_ENABLE_TLS"); emailEnableTLS != "" {
+			cfg.Email.EnableTLS = strings.ToLower(emailEnableTLS) == "true"
+		}
+		if emailRequireAuth := os.Getenv("EMAIL_REQUIRE_AUTH"); emailRequireAuth != "" {
+			cfg.Email.RequireAuth = strings.ToLower(emailRequireAuth) == "true"
+		}
+	} else if !os.IsNotExist(err) {
+		return nil, err
+	} else {
+		// Create default .env file in root directory if it doesn't exist
+		envContent := []string{
+			"SERVER_ADDRESS=" + cfg.ServerAddress,
+			"DATA_DIR=" + cfg.DataDir,
+			"BACKUP_DIR=" + cfg.BackupDir,
+			"JWT_SECRET=" + cfg.JWTSecret,
+			"BASE_URL=" + cfg.BaseURL,
+			"",
+			"# Email configuration",
+			"EMAIL_ENABLED=" + strconv.FormatBool(cfg.Email.Enabled),
+			"EMAIL_HOST=" + cfg.Email.Host,
+			"EMAIL_PORT=" + strconv.Itoa(cfg.Email.Port),
+			"EMAIL_FROM_EMAIL=" + cfg.Email.FromEmail,
+			"EMAIL_FROM_NAME=" + cfg.Email.FromName,
+			"EMAIL_REPLY_TO=" + cfg.Email.ReplyTo,
+			"EMAIL_ENABLE_TLS=" + strconv.FormatBool(cfg.Email.EnableTLS),
+			"EMAIL_REQUIRE_AUTH=" + strconv.FormatBool(cfg.Email.RequireAuth),
+			"EMAIL_USERNAME=" + cfg.Email.Username,
+			"EMAIL_PASSWORD=" + cfg.Email.Password,
+		}
+
+		if err := os.WriteFile(envPath, []byte(strings.Join(envContent, "\n")), 0644); err != nil {
 			return nil, err
 		}
 	}

internal/db/db.go

@@ -102,6 +102,7 @@ type TransferConfig struct {
 	ArchiveEnabled      bool   `gorm:"default:false" form:"archive_enabled"`
 	RcloneFlags         string `form:"rclone_flags"`
 	DeleteAfterTransfer bool   `gorm:"default:false" form:"delete_after_transfer"`
+	SkipProcessedFiles  bool   `gorm:"default:true" form:"skip_processed_files"`
 	CreatedBy           uint
 	User                User `gorm:"foreignkey:CreatedBy"`
 	CreatedAt           time.Time
@@ -378,10 +379,21 @@ func (db *DB) DeleteFileMetadata(id uint) error {
 	return db.Delete(&FileMetadata{}, id).Error
 }
 
+// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
 func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
 	return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
 }
 
+// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
+func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
+	var value bool
+	err := db.Model(&TransferConfig{}).
+		Where("id = ?", configID).
+		Select("skip_processed_files").
+		Scan(&value).Error
+	return value, err
+}
+
 func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 	configPath := db.GetConfigRclonePath(config)
 

internal/db/migrations/add_skip_processed_files.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddSkipProcessedFilesColumn adds the skip_processed_files column to transfer_configs table
+func AddSkipProcessedFilesColumn() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "20250310_add_skip_processed_files",
+		Migrate: func(tx *gorm.DB) error {
+			// Add skip_processed_files column with default value of true
+			return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN skip_processed_files BOOLEAN DEFAULT true").Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the column if needed
+			return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN skip_processed_files").Error
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -11,7 +11,8 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		// ... existing migrations
 		AddDeleteAfterTransferColumn(),
 		AddCloudStorageFields(),
+		AddSkipProcessedFilesColumn(),
 	}
-	
+
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
-} 
+}

internal/email/email.go

@@ -26,12 +26,12 @@ func NewService(cfg *config.Config) *Service {
 func (s *Service) SendPasswordResetEmail(toEmail, username, resetToken string) error {
 	if !s.Config.Email.Enabled {
 		// If email is not enabled, just log it (you can redirect to the default logging logic)
-		return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s", 
+		return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s",
 			s.Config.BaseURL, resetToken)
 	}
 
 	resetLink := fmt.Sprintf("%s/reset-password?token=%s", s.Config.BaseURL, resetToken)
-	
+
 	// Create email data for template
 	data := map[string]interface{}{
 		"Username":     username,
@@ -124,7 +124,7 @@ func (s *Service) generatePasswordResetEmailHTML(data map[string]interface{}) (s
             text-align: center;
         }
         .btn:hover {
-            background-color: #4338ca;
+            background-color:rgb(55, 113, 236);
         }
         .reset-link {
             margin: 20px 0;
@@ -210,7 +210,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
 	headers["Subject"] = subject
 	headers["MIME-Version"] = "1.0"
 	headers["Content-Type"] = "text/html; charset=UTF-8"
-	
+
 	if s.Config.Email.ReplyTo != "" {
 		headers["Reply-To"] = s.Config.Email.ReplyTo
 	}
@@ -224,7 +224,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
 
 	// Set up the SMTP server address
 	addr := fmt.Sprintf("%s:%d", s.Config.Email.Host, s.Config.Email.Port)
-	
+
 	// Check if authentication is required
 	if s.Config.Email.RequireAuth {
 		// Use authenticated SMTP
@@ -237,7 +237,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
 			return fmt.Errorf("failed to connect to SMTP server: %v", err)
 		}
 		defer client.Close()
-		
+
 		// Set up TLS if enabled
 		if s.Config.Email.EnableTLS {
 			if err := client.StartTLS(nil); err != nil {
@@ -252,7 +252,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
 		if err := client.Rcpt(toEmail); err != nil {
 			return fmt.Errorf("failed to set recipient: %v", err)
 		}
-		
+
 		// Send the email body
 		w, err := client.Data()
 		if err != nil {
@@ -266,7 +266,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
 		if err != nil {
 			return fmt.Errorf("failed to close data writer: %v", err)
 		}
-		
+
 		return client.Quit()
 	}
-} 
+}

internal/scheduler/scheduler.go

@@ -128,12 +128,23 @@ func (s *Scheduler) executeJob(jobID uint) {
 		return
 	}
 
-	fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s\n",
+	// Add explicit database reload of the config to ensure we have the latest values
+	var config db.TransferConfig
+	if err := s.db.First(&config, job.Config.ID).Error; err != nil {
+		fmt.Printf("Error loading config %d: %v\n", job.Config.ID, err)
+		return
+	}
+	// Replace the job's config with the freshly loaded one
+	job.Config = config
+
+	// Now the rest of your code will use the correct value
+	fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s, skipProcessedFiles=%v\n",
 		jobID,
 		job.Config.SourceType,
 		job.Config.SourcePath,
 		job.Config.DestinationType,
 		job.Config.DestinationPath,
+		job.Config.SkipProcessedFiles,
 	)
 
 	// Create job history entry
@@ -157,6 +168,12 @@ func (s *Scheduler) executeJob(jobID uint) {
 		fmt.Printf("Error updating job last run time for job %d: %v\n", jobID, err)
 	}
 
+	// Reload the job from the database to get the latest values
+	if err := s.db.Preload("Config").First(&job, jobID).Error; err != nil {
+		fmt.Printf("Error reloading job %d: %v\n", jobID, err)
+		return
+	}
+
 	// Track files already processed in this job execution to prevent duplicates
 	processedFiles := make(map[string]bool)
 
@@ -317,6 +334,8 @@ func (s *Scheduler) executeJob(jobID uint) {
 				}
 			}
 
+			skipFiles := job.Config.SkipProcessedFiles
+
 			// Check if this file has been processed before (by hash)
 			if fileHash != "" {
 				processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
@@ -324,13 +343,22 @@ func (s *Scheduler) executeJob(jobID uint) {
 					fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
 						fileName, fileHash, prevMetadata.FileName)
 
-					// Skip previously processed files with the same hash if they were processed successfully
-					if prevMetadata.Status == "processed" ||
-						prevMetadata.Status == "archived" ||
-						prevMetadata.Status == "deleted" ||
-						prevMetadata.Status == "archived_and_deleted" {
+					// Determine if we should skip this file
+					shouldSkip := false
+					if skipFiles {
+						if prevMetadata.Status == "processed" ||
+							prevMetadata.Status == "archived" ||
+							prevMetadata.Status == "deleted" ||
+							prevMetadata.Status == "archived_and_deleted" {
+							shouldSkip = true
+						}
+					}
+
+					if shouldSkip {
 						fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName)
 						continue
+					} else {
+						fmt.Printf("Re-processing file %s despite previous processing (skipProcessedFiles=%v)\n", fileName, skipFiles)
 					}
 				}
 			}
@@ -341,18 +369,24 @@ func (s *Scheduler) executeJob(jobID uint) {
 				fmt.Printf("File %s was previously processed on %s with status: %s\n",
 					fileName, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
 
-				// If the file was previously processed successfully and the hash hasn't changed,
-				// we could skip processing
-				if prevMetadata.Status == "processed" ||
-					prevMetadata.Status == "archived" ||
-					prevMetadata.Status == "deleted" ||
-					prevMetadata.Status == "archived_and_deleted" {
-					if fileHash != "" && fileHash == prevMetadata.FileHash {
-						fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName)
-						// Skip this file and continue to the next one
-						continue
+				// Determine if we should skip this file based on name+hash match
+				shouldSkip := false
+				if skipFiles && fileHash != "" && fileHash == prevMetadata.FileHash {
+					if prevMetadata.Status == "processed" ||
+						prevMetadata.Status == "archived" ||
+						prevMetadata.Status == "deleted" ||
+						prevMetadata.Status == "archived_and_deleted" {
+						shouldSkip = true
 					}
 				}
+
+				if shouldSkip {
+					fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName)
+					// Skip this file and continue to the next one
+					continue
+				} else if fileHash != "" && fileHash == prevMetadata.FileHash {
+					fmt.Printf("Re-processing file %s despite matching hash (skipProcessedFiles=%v)\n", fileName, skipFiles)
+				}
 			}
 
 			// Prepare moveto command for transfer