diff --git a/.gitignore b/.gitignore index eb63246..8257975 100644 --- a/.gitignore +++ b/.gitignore @@ -58,6 +58,10 @@ components/providers/common/*.go # Ignore the data directory data/ +# Ignore .env files +.env +*.env + # Ignore the tmp directory tmp/ diff --git a/README.md b/README.md index b317aef..fb1a042 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,6 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging - **Multiple Storage Support**: Leverage rclone's extensive support for cloud storage providers: - Amazon S3 - MinIO - - Backblaze B2 - NextCloud - WebDAV - SFTP @@ -114,7 +113,7 @@ docker run -d \ #### Docker Compose Example -For production deployments, you can use Docker Compose: +For production deployments, you can use Docker Compose with environment variables: ```yaml version: '3' @@ -130,6 +129,39 @@ services: - ./backups:/app/data/gomft/backups environment: - TZ=UTC + - SERVER_ADDRESS=:8080 + - DATA_DIR=/app/data/gomft + - BACKUP_DIR=/app/data/gomft/backups + - JWT_SECRET=change_this_to_a_secure_random_string + - BASE_URL=http://localhost:8080 + - EMAIL_ENABLED=true + - EMAIL_HOST=smtp.example.com + - EMAIL_PORT=587 + - EMAIL_FROM_EMAIL=gomft@example.com + - EMAIL_FROM_NAME=GoMFT + - EMAIL_ENABLE_TLS=true + - EMAIL_REQUIRE_AUTH=true + - EMAIL_USERNAME=smtp_username + - EMAIL_PASSWORD=smtp_password +``` + +Alternatively, you can mount your own .env file to the container: + +```yaml +version: '3' +services: + gomft: + image: starfleetcptn/gomft:latest + container_name: gomft + restart: unless-stopped + ports: + - "8080:8080" + volumes: + - ./data:/app/data + - ./backups:/app/data/gomft/backups + - ./.env:/app/.env + environment: + - TZ=UTC ``` Save this as `docker-compose.yml` and run: @@ -142,48 +174,46 @@ For more information and available tags, visit the [GoMFT Docker Hub page](https ## Configuration -GoMFT uses a configuration file located at `./data/gomft/config.json`. On first run, a default configuration will be created: +GoMFT uses an environment file located at `.env` in the root directory of the application. On first run, a default configuration will be created: -```json -{ - "server_address": ":8080", - "data_dir": "./data/gomft", - "backup_dir": "./data/gomft/backups", - "jwt_secret": "your-secret-key", - "base_url": "http://localhost:8080", - "email": { - "enabled": false, - "host": "smtp.example.com", - "port": 587, - "username": "user@example.com", - "password": "your-password", - "from_email": "gomft@example.com", - "from_name": "GoMFT", - "reply_to": "", - "enable_tls": true, - "require_auth": true - } -} +``` +SERVER_ADDRESS=:8080 +DATA_DIR=./data/gomft +BACKUP_DIR=./data/gomft/backups +JWT_SECRET=change_this_to_a_secure_random_string +BASE_URL=http://localhost:8080 + +# Email configuration +EMAIL_ENABLED=true +EMAIL_HOST=smtp.example.com +EMAIL_PORT=587 +EMAIL_FROM_EMAIL=gomft@example.com +EMAIL_FROM_NAME=GoMFT +EMAIL_REPLY_TO= +EMAIL_ENABLE_TLS=true +EMAIL_REQUIRE_AUTH=true +EMAIL_USERNAME=smtp_username +EMAIL_PASSWORD=smtp_password ``` ### Configuration Options -- `server_address`: The address and port to run the server on -- `data_dir`: Directory for storing application data -- `backup_dir`: Directory for storing database backups -- `jwt_secret`: Secret key for JWT token generation -- `base_url`: Base URL for generating links in emails (e.g., password reset links) -- `email`: Email configuration settings for system notifications and password resets - - `enabled`: Set to `true` to enable email functionality - - `host`: SMTP server hostname - - `port`: SMTP server port (usually 587 for TLS, 465 for SSL, or 25 for non-secure) - - `username`: Username for SMTP authentication - - `password`: Password for SMTP authentication - - `from_email`: Email address used as sender - - `from_name`: Name displayed as the sender - - `reply_to`: Optional reply-to email address - - `enable_tls`: Set to `true` to use TLS for secure email transmission - - `require_auth`: Set to `true` to require authentication for SMTP connections, or `false` for servers that don't need authentication +- `SERVER_ADDRESS`: The address and port to run the server on +- `DATA_DIR`: Directory for storing application data +- `BACKUP_DIR`: Directory for storing database backups +- `JWT_SECRET`: Secret key for JWT token generation +- `BASE_URL`: Base URL for generating links in emails (e.g., password reset links) +- Email configuration settings for system notifications and password resets: + - `EMAIL_ENABLED`: Set to `true` to enable email functionality + - `EMAIL_HOST`: SMTP server hostname + - `EMAIL_PORT`: SMTP server port (usually 587 for TLS, 465 for SSL, or 25 for non-secure) + - `EMAIL_USERNAME`: Username for SMTP authentication + - `EMAIL_PASSWORD`: Password for SMTP authentication + - `EMAIL_FROM_EMAIL`: Email address used as sender + - `EMAIL_FROM_NAME`: Name displayed as the sender + - `EMAIL_REPLY_TO`: Optional reply-to email address + - `EMAIL_ENABLE_TLS`: Set to `true` to use TLS for secure email transmission + - `EMAIL_REQUIRE_AUTH`: Set to `true` to require authentication for SMTP connections, or `false` for servers that don't need authentication ## Usage @@ -284,9 +314,9 @@ GoMFT supports email notifications for various features: To configure email functionality: -1. Edit the `config.json` file and provide your SMTP server details -2. Set `"enabled": true` in the email configuration section -3. Ensure the `base_url` setting is configured correctly for your deployment +1. Edit the `.env` file and provide your SMTP server details +2. Set `EMAIL_ENABLED=true` in the email configuration section +3. Ensure the `BASE_URL` setting is configured correctly for your deployment ## Development diff --git a/components/config_form.templ b/components/config_form.templ index 72b90a3..5d28504 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -72,6 +72,7 @@ func getInitialData(config *db.TransferConfig) string { archivePath := "" archiveEnabled := false deleteAfterTransfer := false + skipProcessedFiles := true rcloneFlags := "" // If editing an existing config, populate with those values @@ -129,6 +130,7 @@ func getInitialData(config *db.TransferConfig) string { archivePath = config.ArchivePath archiveEnabled = config.ArchiveEnabled deleteAfterTransfer = config.DeleteAfterTransfer + skipProcessedFiles = config.SkipProcessedFiles rcloneFlags = config.RcloneFlags } @@ -183,6 +185,7 @@ func getInitialData(config *db.TransferConfig) string { archivePath: '%s', archiveEnabled: %v, deleteAfterTransfer: %v, + skipProcessedFiles: %v, rcloneFlags: '%s', }`, name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType, @@ -192,7 +195,7 @@ func getInitialData(config *db.TransferConfig) string { destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType, destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode, destClientId, destClientSecret, destDriveId, destTeamDrive, - archivePath, archiveEnabled, deleteAfterTransfer, rcloneFlags) + archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, rcloneFlags) } templ ConfigForm(ctx context.Context, data ConfigFormData) { diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index f5ddf86..5e51803 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -121,6 +121,26 @@ templ ArchiveOptions() { Warning: This will permanently delete the original files

+ +
+ +

+ Files with the same hash that have been successfully processed before will be skipped +

+
} diff --git a/example.config.json b/example.config.json deleted file mode 100644 index fa032f3..0000000 --- a/example.config.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "server_address": ":8080", - "data_dir": "/app/data/gomft", - "backup_dir": "/app/data/gomft/backups", - "jwt_secret": "change_this_to_a_secure_random_string", - "email": { - "enabled": true, - "host": "smtp.example.com", - "port": 587, - "from_email": "gomft@example.com", - "from_name": "GoMFT", - "reply_to": "", - "enable_tls": true, - "require_auth": true, - "username": "smtp_username", - "password": "smtp_password" - } -} \ No newline at end of file diff --git a/go.mod b/go.mod index 1d0621e..8335dce 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/google/uuid v1.3.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/joho/godotenv v1.5.1 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect diff --git a/go.sum b/go.sum index 74630b2..8a7f379 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= diff --git a/internal/config/config.go b/internal/config/config.go index 245c9b9..5361abe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,31 +1,34 @@ package config import ( - "encoding/json" "os" "path/filepath" + "strconv" + "strings" + + "github.com/joho/godotenv" ) type Config struct { - ServerAddress string `json:"server_address"` - DataDir string `json:"data_dir"` - BackupDir string `json:"backup_dir"` - JWTSecret string `json:"jwt_secret"` + ServerAddress string `json:"server_address"` + DataDir string `json:"data_dir"` + BackupDir string `json:"backup_dir"` + JWTSecret string `json:"jwt_secret"` Email EmailConfig `json:"email"` - BaseURL string `json:"base_url"` // Base URL for generating links in emails + BaseURL string `json:"base_url"` // Base URL for generating links in emails } type EmailConfig struct { - Enabled bool `json:"enabled"` - Host string `json:"host"` - Port int `json:"port"` - Username string `json:"username"` - Password string `json:"password"` - FromEmail string `json:"from_email"` - FromName string `json:"from_name"` - ReplyTo string `json:"reply_to,omitempty"` - EnableTLS bool `json:"enable_tls"` - RequireAuth bool `json:"require_auth"` + Enabled bool `json:"enabled"` + Host string `json:"host"` + Port int `json:"port"` + Username string `json:"username"` + Password string `json:"password"` + FromEmail string `json:"from_email"` + FromName string `json:"from_name"` + ReplyTo string `json:"reply_to,omitempty"` + EnableTLS bool `json:"enable_tls"` + RequireAuth bool `json:"require_auth"` } func Load() (*Config, error) { @@ -37,48 +40,106 @@ func Load() (*Config, error) { JWTSecret: "change_this_to_a_secure_random_string", BaseURL: "http://localhost:8080", Email: EmailConfig{ - Enabled: false, - Host: "smtp.example.com", - Port: 587, - Username: "user@example.com", - Password: "your-password", - FromEmail: "gomft@example.com", - FromName: "GoMFT", - EnableTLS: true, + Enabled: false, + Host: "smtp.example.com", + Port: 587, + Username: "user@example.com", + Password: "your-password", + FromEmail: "gomft@example.com", + FromName: "GoMFT", + EnableTLS: true, RequireAuth: true, }, } - // Check if config file exists - configPath := filepath.Join(cfg.DataDir, "config.json") - if _, err := os.Stat(configPath); err == nil { - // Read configuration file - data, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } - - // Parse configuration - if err := json.Unmarshal(data, cfg); err != nil { - return nil, err - } - } else if !os.IsNotExist(err) { - return nil, err - } - // Ensure data directory exists if err := os.MkdirAll(cfg.DataDir, 0755); err != nil { return nil, err } - // Save configuration if it doesn't exist - if _, err := os.Stat(configPath); os.IsNotExist(err) { - data, err := json.MarshalIndent(cfg, "", " ") - if err != nil { + // First try to load .env from the root directory + envPath := ".env" + if _, err := os.Stat(envPath); err == nil { + // Load .env file + if err := godotenv.Load(envPath); err != nil { return nil, err } - if err := os.WriteFile(configPath, data, 0644); err != nil { + // Override configuration with environment variables + if serverAddr := os.Getenv("SERVER_ADDRESS"); serverAddr != "" { + cfg.ServerAddress = serverAddr + } + if dataDir := os.Getenv("DATA_DIR"); dataDir != "" { + cfg.DataDir = dataDir + } + if backupDir := os.Getenv("BACKUP_DIR"); backupDir != "" { + cfg.BackupDir = backupDir + } + if jwtSecret := os.Getenv("JWT_SECRET"); jwtSecret != "" { + cfg.JWTSecret = jwtSecret + } + if baseURL := os.Getenv("BASE_URL"); baseURL != "" { + cfg.BaseURL = baseURL + } + + // Email configuration + if emailEnabled := os.Getenv("EMAIL_ENABLED"); emailEnabled != "" { + cfg.Email.Enabled = strings.ToLower(emailEnabled) == "true" + } + if emailHost := os.Getenv("EMAIL_HOST"); emailHost != "" { + cfg.Email.Host = emailHost + } + if emailPort := os.Getenv("EMAIL_PORT"); emailPort != "" { + if port, err := strconv.Atoi(emailPort); err == nil { + cfg.Email.Port = port + } + } + if emailUsername := os.Getenv("EMAIL_USERNAME"); emailUsername != "" { + cfg.Email.Username = emailUsername + } + if emailPassword := os.Getenv("EMAIL_PASSWORD"); emailPassword != "" { + cfg.Email.Password = emailPassword + } + if emailFromEmail := os.Getenv("EMAIL_FROM_EMAIL"); emailFromEmail != "" { + cfg.Email.FromEmail = emailFromEmail + } + if emailFromName := os.Getenv("EMAIL_FROM_NAME"); emailFromName != "" { + cfg.Email.FromName = emailFromName + } + if emailReplyTo := os.Getenv("EMAIL_REPLY_TO"); emailReplyTo != "" { + cfg.Email.ReplyTo = emailReplyTo + } + if emailEnableTLS := os.Getenv("EMAIL_ENABLE_TLS"); emailEnableTLS != "" { + cfg.Email.EnableTLS = strings.ToLower(emailEnableTLS) == "true" + } + if emailRequireAuth := os.Getenv("EMAIL_REQUIRE_AUTH"); emailRequireAuth != "" { + cfg.Email.RequireAuth = strings.ToLower(emailRequireAuth) == "true" + } + } else if !os.IsNotExist(err) { + return nil, err + } else { + // Create default .env file in root directory if it doesn't exist + envContent := []string{ + "SERVER_ADDRESS=" + cfg.ServerAddress, + "DATA_DIR=" + cfg.DataDir, + "BACKUP_DIR=" + cfg.BackupDir, + "JWT_SECRET=" + cfg.JWTSecret, + "BASE_URL=" + cfg.BaseURL, + "", + "# Email configuration", + "EMAIL_ENABLED=" + strconv.FormatBool(cfg.Email.Enabled), + "EMAIL_HOST=" + cfg.Email.Host, + "EMAIL_PORT=" + strconv.Itoa(cfg.Email.Port), + "EMAIL_FROM_EMAIL=" + cfg.Email.FromEmail, + "EMAIL_FROM_NAME=" + cfg.Email.FromName, + "EMAIL_REPLY_TO=" + cfg.Email.ReplyTo, + "EMAIL_ENABLE_TLS=" + strconv.FormatBool(cfg.Email.EnableTLS), + "EMAIL_REQUIRE_AUTH=" + strconv.FormatBool(cfg.Email.RequireAuth), + "EMAIL_USERNAME=" + cfg.Email.Username, + "EMAIL_PASSWORD=" + cfg.Email.Password, + } + + if err := os.WriteFile(envPath, []byte(strings.Join(envContent, "\n")), 0644); err != nil { return nil, err } } diff --git a/internal/db/db.go b/internal/db/db.go index 7a6a12d..c157835 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -102,6 +102,7 @@ type TransferConfig struct { ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"` RcloneFlags string `form:"rclone_flags"` DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"` + SkipProcessedFiles bool `gorm:"default:true" form:"skip_processed_files"` CreatedBy uint User User `gorm:"foreignkey:CreatedBy"` CreatedAt time.Time @@ -378,10 +379,21 @@ func (db *DB) DeleteFileMetadata(id uint) error { return db.Delete(&FileMetadata{}, id).Error } +// GetConfigRclonePath returns the path to the rclone config file for a given transfer config func (db *DB) GetConfigRclonePath(config *TransferConfig) string { return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID)) } +// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config +func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) { + var value bool + err := db.Model(&TransferConfig{}). + Where("id = ?", configID). + Select("skip_processed_files"). + Scan(&value).Error + return value, err +} + func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { configPath := db.GetConfigRclonePath(config) diff --git a/internal/db/migrations/add_skip_processed_files.go b/internal/db/migrations/add_skip_processed_files.go new file mode 100644 index 0000000..f6939b1 --- /dev/null +++ b/internal/db/migrations/add_skip_processed_files.go @@ -0,0 +1,21 @@ +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddSkipProcessedFilesColumn adds the skip_processed_files column to transfer_configs table +func AddSkipProcessedFilesColumn() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20250310_add_skip_processed_files", + Migrate: func(tx *gorm.DB) error { + // Add skip_processed_files column with default value of true + return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN skip_processed_files BOOLEAN DEFAULT true").Error + }, + Rollback: func(tx *gorm.DB) error { + // Drop the column if needed + return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN skip_processed_files").Error + }, + } +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index c7705a7..49e99a9 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -11,7 +11,8 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate { // ... existing migrations AddDeleteAfterTransferColumn(), AddCloudStorageFields(), + AddSkipProcessedFilesColumn(), } - + return gormigrate.New(db, gormigrate.DefaultOptions, migrations) -} \ No newline at end of file +} diff --git a/internal/email/email.go b/internal/email/email.go index bcc969d..7d7b9da 100644 --- a/internal/email/email.go +++ b/internal/email/email.go @@ -26,12 +26,12 @@ func NewService(cfg *config.Config) *Service { func (s *Service) SendPasswordResetEmail(toEmail, username, resetToken string) error { if !s.Config.Email.Enabled { // If email is not enabled, just log it (you can redirect to the default logging logic) - return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s", + return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s", s.Config.BaseURL, resetToken) } resetLink := fmt.Sprintf("%s/reset-password?token=%s", s.Config.BaseURL, resetToken) - + // Create email data for template data := map[string]interface{}{ "Username": username, @@ -124,7 +124,7 @@ func (s *Service) generatePasswordResetEmailHTML(data map[string]interface{}) (s text-align: center; } .btn:hover { - background-color: #4338ca; + background-color:rgb(55, 113, 236); } .reset-link { margin: 20px 0; @@ -210,7 +210,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error { headers["Subject"] = subject headers["MIME-Version"] = "1.0" headers["Content-Type"] = "text/html; charset=UTF-8" - + if s.Config.Email.ReplyTo != "" { headers["Reply-To"] = s.Config.Email.ReplyTo } @@ -224,7 +224,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error { // Set up the SMTP server address addr := fmt.Sprintf("%s:%d", s.Config.Email.Host, s.Config.Email.Port) - + // Check if authentication is required if s.Config.Email.RequireAuth { // Use authenticated SMTP @@ -237,7 +237,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error { return fmt.Errorf("failed to connect to SMTP server: %v", err) } defer client.Close() - + // Set up TLS if enabled if s.Config.Email.EnableTLS { if err := client.StartTLS(nil); err != nil { @@ -252,7 +252,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error { if err := client.Rcpt(toEmail); err != nil { return fmt.Errorf("failed to set recipient: %v", err) } - + // Send the email body w, err := client.Data() if err != nil { @@ -266,7 +266,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error { if err != nil { return fmt.Errorf("failed to close data writer: %v", err) } - + return client.Quit() } -} \ No newline at end of file +} diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 401221f..2604ebe 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -128,12 +128,23 @@ func (s *Scheduler) executeJob(jobID uint) { return } - fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s\n", + // Add explicit database reload of the config to ensure we have the latest values + var config db.TransferConfig + if err := s.db.First(&config, job.Config.ID).Error; err != nil { + fmt.Printf("Error loading config %d: %v\n", job.Config.ID, err) + return + } + // Replace the job's config with the freshly loaded one + job.Config = config + + // Now the rest of your code will use the correct value + fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s, skipProcessedFiles=%v\n", jobID, job.Config.SourceType, job.Config.SourcePath, job.Config.DestinationType, job.Config.DestinationPath, + job.Config.SkipProcessedFiles, ) // Create job history entry @@ -157,6 +168,12 @@ func (s *Scheduler) executeJob(jobID uint) { fmt.Printf("Error updating job last run time for job %d: %v\n", jobID, err) } + // Reload the job from the database to get the latest values + if err := s.db.Preload("Config").First(&job, jobID).Error; err != nil { + fmt.Printf("Error reloading job %d: %v\n", jobID, err) + return + } + // Track files already processed in this job execution to prevent duplicates processedFiles := make(map[string]bool) @@ -317,6 +334,8 @@ func (s *Scheduler) executeJob(jobID uint) { } } + skipFiles := job.Config.SkipProcessedFiles + // Check if this file has been processed before (by hash) if fileHash != "" { processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash) @@ -324,13 +343,22 @@ func (s *Scheduler) executeJob(jobID uint) { fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n", fileName, fileHash, prevMetadata.FileName) - // Skip previously processed files with the same hash if they were processed successfully - if prevMetadata.Status == "processed" || - prevMetadata.Status == "archived" || - prevMetadata.Status == "deleted" || - prevMetadata.Status == "archived_and_deleted" { + // Determine if we should skip this file + shouldSkip := false + if skipFiles { + if prevMetadata.Status == "processed" || + prevMetadata.Status == "archived" || + prevMetadata.Status == "deleted" || + prevMetadata.Status == "archived_and_deleted" { + shouldSkip = true + } + } + + if shouldSkip { fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName) continue + } else { + fmt.Printf("Re-processing file %s despite previous processing (skipProcessedFiles=%v)\n", fileName, skipFiles) } } } @@ -341,18 +369,24 @@ func (s *Scheduler) executeJob(jobID uint) { fmt.Printf("File %s was previously processed on %s with status: %s\n", fileName, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status) - // If the file was previously processed successfully and the hash hasn't changed, - // we could skip processing - if prevMetadata.Status == "processed" || - prevMetadata.Status == "archived" || - prevMetadata.Status == "deleted" || - prevMetadata.Status == "archived_and_deleted" { - if fileHash != "" && fileHash == prevMetadata.FileHash { - fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName) - // Skip this file and continue to the next one - continue + // Determine if we should skip this file based on name+hash match + shouldSkip := false + if skipFiles && fileHash != "" && fileHash == prevMetadata.FileHash { + if prevMetadata.Status == "processed" || + prevMetadata.Status == "archived" || + prevMetadata.Status == "deleted" || + prevMetadata.Status == "archived_and_deleted" { + shouldSkip = true } } + + if shouldSkip { + fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName) + // Skip this file and continue to the next one + continue + } else if fileHash != "" && fileHash == prevMetadata.FileHash { + fmt.Printf("Re-processing file %s despite matching hash (skipProcessedFiles=%v)\n", fileName, skipFiles) + } } // Prepare moveto command for transfer diff --git a/screenshots/file.metadata.gomft.png b/screenshots/file.metadata.gomft.png new file mode 100644 index 0000000..129092d Binary files /dev/null and b/screenshots/file.metadata.gomft.png differ