feat: Add support for skipping processed files in transfer configs

- Introduce new `skip_processed_files` option for transfer configurations
- Add database migration to support the new column
- Update config form and UI to include skip processed files toggle
- Enhance scheduler to respect skip processed files setting
- Modify file processing logic to optionally skip previously processed files
- Add environment variable support for skip processed files configuration

internal/db/db.go

@@ -102,6 +102,7 @@ type TransferConfig struct {
 	ArchiveEnabled      bool   `gorm:"default:false" form:"archive_enabled"`
 	RcloneFlags         string `form:"rclone_flags"`
 	DeleteAfterTransfer bool   `gorm:"default:false" form:"delete_after_transfer"`
+	SkipProcessedFiles  bool   `gorm:"default:true" form:"skip_processed_files"`
 	CreatedBy           uint
 	User                User `gorm:"foreignkey:CreatedBy"`
 	CreatedAt           time.Time
@@ -378,10 +379,21 @@ func (db *DB) DeleteFileMetadata(id uint) error {
 	return db.Delete(&FileMetadata{}, id).Error
 }
 
+// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
 func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
 	return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
 }
 
+// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
+func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
+	var value bool
+	err := db.Model(&TransferConfig{}).
+		Where("id = ?", configID).
+		Select("skip_processed_files").
+		Scan(&value).Error
+	return value, err
+}
+
 func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 	configPath := db.GetConfigRclonePath(config)
 

internal/db/migrations/add_skip_processed_files.go

@@ -0,0 +1,21 @@
+package migrations
+
+import (
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+// AddSkipProcessedFilesColumn adds the skip_processed_files column to transfer_configs table
+func AddSkipProcessedFilesColumn() *gormigrate.Migration {
+	return &gormigrate.Migration{
+		ID: "20250310_add_skip_processed_files",
+		Migrate: func(tx *gorm.DB) error {
+			// Add skip_processed_files column with default value of true
+			return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN skip_processed_files BOOLEAN DEFAULT true").Error
+		},
+		Rollback: func(tx *gorm.DB) error {
+			// Drop the column if needed
+			return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN skip_processed_files").Error
+		},
+	}
+}

internal/db/migrations/migrations.go

@@ -11,7 +11,8 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
 		// ... existing migrations
 		AddDeleteAfterTransferColumn(),
 		AddCloudStorageFields(),
+		AddSkipProcessedFilesColumn(),
 	}
-	
+
 	return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
-} 
+}