+
+}
+
+templ FilePatternFields() {
+
+
+
File
+ Pattern
+
+
+ Glob pattern for files to transfer. Leave empty to transfer all files.
+
+
+
+
+
Output Pattern
+
+
+ Pattern for output filename. Use variables like ${`filename`}, ${`timestamp`}, ${`date`}
+
+
+
+ Pattern for filenames. Available variables:
+ ${`filename`} - Original filename without extension (e.g., "report")
+ ${`ext`} - Original file extension (e.g., "csv")
+ ${`date:format`} - Current date using Go's time format:
+ • 2006-01-02 → YYYY-MM-DD
+ • 20060102 → YYYYMMDD
+ • 2006-01-02 15:04:05 → YYYY-MM-DD_HH:MM:SS
+ Example: ${`filename`}_${`date:2006-01-02`}_${`ext`} → "report_2023-03-01.csv"
+
+
+
+
+}
+
+templ ArchiveOptions() {
+
+
Archive & Delete Options
+
+
+
+
+
+ Enable archiving
+
+
+
+
+
+
Archive Path
+
+
+
+
+
+
+
+ Files will be moved here after successful transfer
+
+
+
+
+
+
+
+ Delete source files after transfer
+
+
+
+ Warning: This will permanently delete the original files
+
+
+
+
+
+
+
+ Skip files that have already been processed
+
+
+
+ Files with the same hash that have been successfully processed before will be skipped
+
+
+
+}
+
+templ RcloneFlags() {
+
+
Rclone Flags
+
+
+ Optional: Additional rclone flags for fine-tuning the transfer.
+
+
+}
+
+templ SourceSelection() {
+
+
Source
+ Type
+
+
+
+
+
+ Local
+ SFTP
+ FTP
+ S3
+ MinIO
+ SMB
+ NextCloud
+ WebDAV
+
+
+
+}
+
+templ DestinationSelection() {
+
+
Destination Type
+
+
+
+
+
+ Local
+ SFTP
+ FTP
+ S3
+ MinIO
+ SMB
+ NextCloud
+ WebDAV
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/destination.go b/components/providers/destination/destination.go
new file mode 100644
index 0000000..e84df9b
--- /dev/null
+++ b/components/providers/destination/destination.go
@@ -0,0 +1,3 @@
+// Package destination contains the UI destination forms for the GoMFT application.
+// This file serves as a marker for the destination package to ensure it's properly recognized by Go.
+package destination
diff --git a/components/providers/destination/ftp.templ b/components/providers/destination/ftp.templ
new file mode 100644
index 0000000..391b7bb
--- /dev/null
+++ b/components/providers/destination/ftp.templ
@@ -0,0 +1,106 @@
+package destination
+
+templ FTPDestinationForm() {
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Use Passive Mode
+
Enable passive mode for FTP connection
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/local.templ b/components/providers/destination/local.templ
new file mode 100644
index 0000000..deb7465
--- /dev/null
+++ b/components/providers/destination/local.templ
@@ -0,0 +1,22 @@
+package destination
+
+templ LocalDestinationForm() {
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/minio.templ b/components/providers/destination/minio.templ
new file mode 100644
index 0000000..dfbcac2
--- /dev/null
+++ b/components/providers/destination/minio.templ
@@ -0,0 +1,91 @@
+package destination
+
+templ MinIODestinationForm() {
+
+
+
+
+
+
+
Path Prefix
+
+
+ Optional. If specified, files will be uploaded to this path in the bucket.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/nextcloud.templ b/components/providers/destination/nextcloud.templ
new file mode 100644
index 0000000..9cc4daf
--- /dev/null
+++ b/components/providers/destination/nextcloud.templ
@@ -0,0 +1,56 @@
+package destination
+
+templ NextCloudDestinationForm() {
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/s3.templ b/components/providers/destination/s3.templ
new file mode 100644
index 0000000..90ba488
--- /dev/null
+++ b/components/providers/destination/s3.templ
@@ -0,0 +1,91 @@
+package destination
+
+templ S3DestinationForm() {
+
+
+
+
+
+
+
S3 Path Prefix
+
+
+ Optional. If specified, files will be uploaded to this path in the bucket.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/sftp.templ b/components/providers/destination/sftp.templ
new file mode 100644
index 0000000..e224d59
--- /dev/null
+++ b/components/providers/destination/sftp.templ
@@ -0,0 +1,118 @@
+package destination
+
+templ SFTPDestinationForm() {
+
+
+
+
+
+
+
+
+
+
+
+
Use Password
+
+
+
+
Use Key File
+
+
+
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/smb.templ b/components/providers/destination/smb.templ
new file mode 100644
index 0000000..515810e
--- /dev/null
+++ b/components/providers/destination/smb.templ
@@ -0,0 +1,111 @@
+package destination
+
+templ SMBDestinationForm() {
+
+
+
+
+
+
+
Path
+
+
+ Optional. Subdirectory within the share.
+
+
+
+
+
Domain
+
+
+ Optional. Windows domain for authentication.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/destination/webdav.templ b/components/providers/destination/webdav.templ
new file mode 100644
index 0000000..3489cd4
--- /dev/null
+++ b/components/providers/destination/webdav.templ
@@ -0,0 +1,75 @@
+package destination
+
+templ WebDAVDestinationForm() {
+
+
+
+
+
Remote Path
+
+
+ Optional. Relative to the WebDAV URL.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/providers.go b/components/providers/providers.go
new file mode 100644
index 0000000..bb77876
--- /dev/null
+++ b/components/providers/providers.go
@@ -0,0 +1,3 @@
+// Package providers contains the UI providers for the GoMFT application.
+// This file serves as a marker for the providers package to ensure it's properly recognized by Go.
+package providers
diff --git a/components/providers/source/ftp.templ b/components/providers/source/ftp.templ
new file mode 100644
index 0000000..201ba59
--- /dev/null
+++ b/components/providers/source/ftp.templ
@@ -0,0 +1,106 @@
+package source
+
+templ FTPSourceForm() {
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Use Passive Mode
+
Enable passive mode for FTP connection
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/local.templ b/components/providers/source/local.templ
new file mode 100644
index 0000000..f61fc73
--- /dev/null
+++ b/components/providers/source/local.templ
@@ -0,0 +1,22 @@
+package source
+
+templ LocalSourceForm() {
+
+}
\ No newline at end of file
diff --git a/components/providers/source/minio.templ b/components/providers/source/minio.templ
new file mode 100644
index 0000000..1e4553c
--- /dev/null
+++ b/components/providers/source/minio.templ
@@ -0,0 +1,91 @@
+package source
+
+templ MinIOSourceForm() {
+
+
+
+
+
+
+
Path Prefix
+
+
+ Optional. If specified, only files in this path will be processed.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/nextcloud.templ b/components/providers/source/nextcloud.templ
new file mode 100644
index 0000000..420f767
--- /dev/null
+++ b/components/providers/source/nextcloud.templ
@@ -0,0 +1,75 @@
+package source
+
+templ NextCloudSourceForm() {
+
+
+
+
+
Remote Path
+
+
+ Optional. If left empty, root directory will be used.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/s3.templ b/components/providers/source/s3.templ
new file mode 100644
index 0000000..2794dfb
--- /dev/null
+++ b/components/providers/source/s3.templ
@@ -0,0 +1,91 @@
+package source
+
+templ S3SourceForm() {
+
+
+
+
+
+
+
S3 Path Prefix
+
+
+ Optional. If specified, only files in this path will be processed.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/sftp.templ b/components/providers/source/sftp.templ
new file mode 100644
index 0000000..ab0bb88
--- /dev/null
+++ b/components/providers/source/sftp.templ
@@ -0,0 +1,135 @@
+package source
+
+templ SFTPSourceForm() {
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Use Password
+
+
+
+
Use Key File
+
+
+
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/smb.templ b/components/providers/source/smb.templ
new file mode 100644
index 0000000..651a960
--- /dev/null
+++ b/components/providers/source/smb.templ
@@ -0,0 +1,111 @@
+package source
+
+templ SMBSourceForm() {
+
+
+
+
+
+
+
Path
+
+
+ Optional. Subdirectory within the share.
+
+
+
+
+
Domain
+
+
+ Optional. Windows domain for authentication.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/components/providers/source/source.go b/components/providers/source/source.go
new file mode 100644
index 0000000..5d10428
--- /dev/null
+++ b/components/providers/source/source.go
@@ -0,0 +1,3 @@
+// Package source contains the UI source forms for the GoMFT application.
+// This file serves as a marker for the source package to ensure it's properly recognized by Go.
+package source
diff --git a/components/providers/source/webdav.templ b/components/providers/source/webdav.templ
new file mode 100644
index 0000000..17d9c06
--- /dev/null
+++ b/components/providers/source/webdav.templ
@@ -0,0 +1,75 @@
+package source
+
+templ WebDAVSourceForm() {
+
+
+
+
+
Remote Path
+
+
+ Optional. Relative to the WebDAV URL.
+
+
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/example.config.json b/example.config.json
deleted file mode 100644
index fa032f3..0000000
--- a/example.config.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
- "server_address": ":8080",
- "data_dir": "/app/data/gomft",
- "backup_dir": "/app/data/gomft/backups",
- "jwt_secret": "change_this_to_a_secure_random_string",
- "email": {
- "enabled": true,
- "host": "smtp.example.com",
- "port": 587,
- "from_email": "gomft@example.com",
- "from_name": "GoMFT",
- "reply_to": "",
- "enable_tls": true,
- "require_auth": true,
- "username": "smtp_username",
- "password": "smtp_password"
- }
-}
\ No newline at end of file
diff --git a/go.mod b/go.mod
index 1d0621e..8335dce 100644
--- a/go.mod
+++ b/go.mod
@@ -28,6 +28,7 @@ require (
github.com/google/uuid v1.3.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
+ github.com/joho/godotenv v1.5.1 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.10 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
diff --git a/go.sum b/go.sum
index 74630b2..8a7f379 100644
--- a/go.sum
+++ b/go.sum
@@ -48,6 +48,8 @@ github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
diff --git a/internal/config/config.go b/internal/config/config.go
index 245c9b9..5361abe 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -1,31 +1,34 @@
package config
import (
- "encoding/json"
"os"
"path/filepath"
+ "strconv"
+ "strings"
+
+ "github.com/joho/godotenv"
)
type Config struct {
- ServerAddress string `json:"server_address"`
- DataDir string `json:"data_dir"`
- BackupDir string `json:"backup_dir"`
- JWTSecret string `json:"jwt_secret"`
+ ServerAddress string `json:"server_address"`
+ DataDir string `json:"data_dir"`
+ BackupDir string `json:"backup_dir"`
+ JWTSecret string `json:"jwt_secret"`
Email EmailConfig `json:"email"`
- BaseURL string `json:"base_url"` // Base URL for generating links in emails
+ BaseURL string `json:"base_url"` // Base URL for generating links in emails
}
type EmailConfig struct {
- Enabled bool `json:"enabled"`
- Host string `json:"host"`
- Port int `json:"port"`
- Username string `json:"username"`
- Password string `json:"password"`
- FromEmail string `json:"from_email"`
- FromName string `json:"from_name"`
- ReplyTo string `json:"reply_to,omitempty"`
- EnableTLS bool `json:"enable_tls"`
- RequireAuth bool `json:"require_auth"`
+ Enabled bool `json:"enabled"`
+ Host string `json:"host"`
+ Port int `json:"port"`
+ Username string `json:"username"`
+ Password string `json:"password"`
+ FromEmail string `json:"from_email"`
+ FromName string `json:"from_name"`
+ ReplyTo string `json:"reply_to,omitempty"`
+ EnableTLS bool `json:"enable_tls"`
+ RequireAuth bool `json:"require_auth"`
}
func Load() (*Config, error) {
@@ -37,48 +40,106 @@ func Load() (*Config, error) {
JWTSecret: "change_this_to_a_secure_random_string",
BaseURL: "http://localhost:8080",
Email: EmailConfig{
- Enabled: false,
- Host: "smtp.example.com",
- Port: 587,
- Username: "user@example.com",
- Password: "your-password",
- FromEmail: "gomft@example.com",
- FromName: "GoMFT",
- EnableTLS: true,
+ Enabled: false,
+ Host: "smtp.example.com",
+ Port: 587,
+ Username: "user@example.com",
+ Password: "your-password",
+ FromEmail: "gomft@example.com",
+ FromName: "GoMFT",
+ EnableTLS: true,
RequireAuth: true,
},
}
- // Check if config file exists
- configPath := filepath.Join(cfg.DataDir, "config.json")
- if _, err := os.Stat(configPath); err == nil {
- // Read configuration file
- data, err := os.ReadFile(configPath)
- if err != nil {
- return nil, err
- }
-
- // Parse configuration
- if err := json.Unmarshal(data, cfg); err != nil {
- return nil, err
- }
- } else if !os.IsNotExist(err) {
- return nil, err
- }
-
// Ensure data directory exists
if err := os.MkdirAll(cfg.DataDir, 0755); err != nil {
return nil, err
}
- // Save configuration if it doesn't exist
- if _, err := os.Stat(configPath); os.IsNotExist(err) {
- data, err := json.MarshalIndent(cfg, "", " ")
- if err != nil {
+ // First try to load .env from the root directory
+ envPath := ".env"
+ if _, err := os.Stat(envPath); err == nil {
+ // Load .env file
+ if err := godotenv.Load(envPath); err != nil {
return nil, err
}
- if err := os.WriteFile(configPath, data, 0644); err != nil {
+ // Override configuration with environment variables
+ if serverAddr := os.Getenv("SERVER_ADDRESS"); serverAddr != "" {
+ cfg.ServerAddress = serverAddr
+ }
+ if dataDir := os.Getenv("DATA_DIR"); dataDir != "" {
+ cfg.DataDir = dataDir
+ }
+ if backupDir := os.Getenv("BACKUP_DIR"); backupDir != "" {
+ cfg.BackupDir = backupDir
+ }
+ if jwtSecret := os.Getenv("JWT_SECRET"); jwtSecret != "" {
+ cfg.JWTSecret = jwtSecret
+ }
+ if baseURL := os.Getenv("BASE_URL"); baseURL != "" {
+ cfg.BaseURL = baseURL
+ }
+
+ // Email configuration
+ if emailEnabled := os.Getenv("EMAIL_ENABLED"); emailEnabled != "" {
+ cfg.Email.Enabled = strings.ToLower(emailEnabled) == "true"
+ }
+ if emailHost := os.Getenv("EMAIL_HOST"); emailHost != "" {
+ cfg.Email.Host = emailHost
+ }
+ if emailPort := os.Getenv("EMAIL_PORT"); emailPort != "" {
+ if port, err := strconv.Atoi(emailPort); err == nil {
+ cfg.Email.Port = port
+ }
+ }
+ if emailUsername := os.Getenv("EMAIL_USERNAME"); emailUsername != "" {
+ cfg.Email.Username = emailUsername
+ }
+ if emailPassword := os.Getenv("EMAIL_PASSWORD"); emailPassword != "" {
+ cfg.Email.Password = emailPassword
+ }
+ if emailFromEmail := os.Getenv("EMAIL_FROM_EMAIL"); emailFromEmail != "" {
+ cfg.Email.FromEmail = emailFromEmail
+ }
+ if emailFromName := os.Getenv("EMAIL_FROM_NAME"); emailFromName != "" {
+ cfg.Email.FromName = emailFromName
+ }
+ if emailReplyTo := os.Getenv("EMAIL_REPLY_TO"); emailReplyTo != "" {
+ cfg.Email.ReplyTo = emailReplyTo
+ }
+ if emailEnableTLS := os.Getenv("EMAIL_ENABLE_TLS"); emailEnableTLS != "" {
+ cfg.Email.EnableTLS = strings.ToLower(emailEnableTLS) == "true"
+ }
+ if emailRequireAuth := os.Getenv("EMAIL_REQUIRE_AUTH"); emailRequireAuth != "" {
+ cfg.Email.RequireAuth = strings.ToLower(emailRequireAuth) == "true"
+ }
+ } else if !os.IsNotExist(err) {
+ return nil, err
+ } else {
+ // Create default .env file in root directory if it doesn't exist
+ envContent := []string{
+ "SERVER_ADDRESS=" + cfg.ServerAddress,
+ "DATA_DIR=" + cfg.DataDir,
+ "BACKUP_DIR=" + cfg.BackupDir,
+ "JWT_SECRET=" + cfg.JWTSecret,
+ "BASE_URL=" + cfg.BaseURL,
+ "",
+ "# Email configuration",
+ "EMAIL_ENABLED=" + strconv.FormatBool(cfg.Email.Enabled),
+ "EMAIL_HOST=" + cfg.Email.Host,
+ "EMAIL_PORT=" + strconv.Itoa(cfg.Email.Port),
+ "EMAIL_FROM_EMAIL=" + cfg.Email.FromEmail,
+ "EMAIL_FROM_NAME=" + cfg.Email.FromName,
+ "EMAIL_REPLY_TO=" + cfg.Email.ReplyTo,
+ "EMAIL_ENABLE_TLS=" + strconv.FormatBool(cfg.Email.EnableTLS),
+ "EMAIL_REQUIRE_AUTH=" + strconv.FormatBool(cfg.Email.RequireAuth),
+ "EMAIL_USERNAME=" + cfg.Email.Username,
+ "EMAIL_PASSWORD=" + cfg.Email.Password,
+ }
+
+ if err := os.WriteFile(envPath, []byte(strings.Join(envContent, "\n")), 0644); err != nil {
return nil, err
}
}
diff --git a/internal/db/db.go b/internal/db/db.go
index 1779f94..c157835 100644
--- a/internal/db/db.go
+++ b/internal/db/db.go
@@ -7,23 +7,23 @@ import (
"path/filepath"
"time"
- "github.com/starfleetcptn/gomft/internal/auth"
"github.com/glebarez/sqlite"
+ "github.com/starfleetcptn/gomft/internal/auth"
"gorm.io/gorm"
)
type User struct {
- ID uint `gorm:"primarykey"`
- Email string `gorm:"unique;not null"`
- PasswordHash string `gorm:"not null"`
- IsAdmin bool `gorm:"default:false"`
- LastPasswordChange time.Time
- FailedLoginAttempts int `gorm:"default:0"`
- AccountLocked bool `gorm:"default:false"`
- LockoutUntil *time.Time
- Theme string `gorm:"default:'light'"`
- CreatedAt time.Time
- UpdatedAt time.Time
+ ID uint `gorm:"primarykey"`
+ Email string `gorm:"unique;not null"`
+ PasswordHash string `gorm:"not null"`
+ IsAdmin bool `gorm:"default:false"`
+ LastPasswordChange time.Time
+ FailedLoginAttempts int `gorm:"default:0"`
+ AccountLocked bool `gorm:"default:false"`
+ LockoutUntil *time.Time
+ Theme string `gorm:"default:'light'"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
}
type PasswordHistory struct {
@@ -46,31 +46,31 @@ type PasswordResetToken struct {
}
type TransferConfig struct {
- ID uint `gorm:"primarykey"`
- Name string `gorm:"not null" form:"name"`
- SourceType string `gorm:"not null" form:"source_type"`
- SourcePath string `gorm:"not null" form:"source_path"`
- SourceHost string `form:"source_host"`
- SourcePort int `gorm:"default:22" form:"source_port"`
- SourceUser string `form:"source_user"`
- SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
- SourceKeyFile string `form:"source_key_file"`
+ ID uint `gorm:"primarykey"`
+ Name string `gorm:"not null" form:"name"`
+ SourceType string `gorm:"not null" form:"source_type"`
+ SourcePath string `gorm:"not null" form:"source_path"`
+ SourceHost string `form:"source_host"`
+ SourcePort int `gorm:"default:22" form:"source_port"`
+ SourceUser string `form:"source_user"`
+ SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
+ SourceKeyFile string `form:"source_key_file"`
// S3 source fields
- SourceBucket string `form:"source_bucket"`
- SourceRegion string `form:"source_region"`
- SourceAccessKey string `form:"source_access_key"`
- SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
- SourceEndpoint string `form:"source_endpoint"`
+ SourceBucket string `form:"source_bucket"`
+ SourceRegion string `form:"source_region"`
+ SourceAccessKey string `form:"source_access_key"`
+ SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
+ SourceEndpoint string `form:"source_endpoint"`
// SMB source fields
- SourceShare string `form:"source_share"`
- SourceDomain string `form:"source_domain"`
+ SourceShare string `form:"source_share"`
+ SourceDomain string `form:"source_domain"`
// FTP source fields
- SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
+ SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
// OneDrive and Google Drive source fields
SourceClientID string `form:"source_client_id"`
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
- SourceDriveID string `form:"source_drive_id"` // For OneDrive
- SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
+ SourceDriveID string `form:"source_drive_id"` // For OneDrive
+ SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
// General fields
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
@@ -82,30 +82,31 @@ type TransferConfig struct {
DestPassword string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
DestKeyFile string `form:"dest_key_file"`
// S3 destination fields
- DestBucket string `form:"dest_bucket"`
- DestRegion string `form:"dest_region"`
- DestAccessKey string `form:"dest_access_key"`
- DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
- DestEndpoint string `form:"dest_endpoint"`
+ DestBucket string `form:"dest_bucket"`
+ DestRegion string `form:"dest_region"`
+ DestAccessKey string `form:"dest_access_key"`
+ DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
+ DestEndpoint string `form:"dest_endpoint"`
// SMB destination fields
- DestShare string `form:"dest_share"`
- DestDomain string `form:"dest_domain"`
+ DestShare string `form:"dest_share"`
+ DestDomain string `form:"dest_domain"`
// FTP destination fields
- DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
+ DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
// OneDrive and Google Drive destination fields
DestClientID string `form:"dest_client_id"`
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
- DestDriveID string `form:"dest_drive_id"` // For OneDrive
- DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
+ DestDriveID string `form:"dest_drive_id"` // For OneDrive
+ DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
// General fields
- ArchivePath string `form:"archive_path"`
- ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
- RcloneFlags string `form:"rclone_flags"`
+ ArchivePath string `form:"archive_path"`
+ ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
+ RcloneFlags string `form:"rclone_flags"`
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
- CreatedBy uint
- User User `gorm:"foreignkey:CreatedBy"`
- CreatedAt time.Time
- UpdatedAt time.Time
+ SkipProcessedFiles bool `gorm:"default:true" form:"skip_processed_files"`
+ CreatedBy uint
+ User User `gorm:"foreignkey:CreatedBy"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
}
type Job struct {
@@ -135,6 +136,25 @@ type JobHistory struct {
ErrorMessage string
}
+// FileMetadata stores information about processed files
+type FileMetadata struct {
+ ID uint `gorm:"primarykey"`
+ JobID uint `gorm:"not null;index"`
+ Job Job `gorm:"foreignkey:JobID"`
+ FileName string `gorm:"not null"`
+ OriginalPath string `gorm:"not null"`
+ FileSize int64 `gorm:"not null"`
+ FileHash string `gorm:"index"` // MD5 or other hash for file identity
+ CreationTime time.Time
+ ModTime time.Time
+ ProcessedTime time.Time `gorm:"not null"`
+ DestinationPath string `gorm:"not null"`
+ Status string `gorm:"not null"` // processed, archived, deleted, etc.
+ ErrorMessage string
+ CreatedAt time.Time
+ UpdatedAt time.Time
+}
+
type DB struct {
*gorm.DB
}
@@ -153,7 +173,7 @@ func Initialize(dbPath string) (*DB, error) {
}
// Auto migrate the schema
- err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{})
+ err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{}, &FileMetadata{})
if err != nil {
return nil, fmt.Errorf("failed to migrate database: %v", err)
}
@@ -302,16 +322,78 @@ func (db *DB) UpdateJobHistory(history *JobHistory) error {
}
func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
- var history []JobHistory
- err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&history).Error
- return history, err
+ var histories []JobHistory
+ err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&histories).Error
+ return histories, err
}
-// Helper functions
+// CreateFileMetadata creates a new file metadata record
+func (db *DB) CreateFileMetadata(metadata *FileMetadata) error {
+ return db.Create(metadata).Error
+}
+
+// GetFileMetadata retrieves file metadata by ID
+func (db *DB) GetFileMetadata(id uint) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.First(&metadata, id).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// GetFileMetadataByJobAndName retrieves file metadata by job ID and filename
+func (db *DB) GetFileMetadataByJobAndName(jobID uint, fileName string) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.Where("job_id = ? AND file_name = ?", jobID, fileName).First(&metadata).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// GetFileMetadataByHash retrieves file metadata by file hash
+func (db *DB) GetFileMetadataByHash(fileHash string) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.Where("file_hash = ?", fileHash).First(&metadata).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// UpdateFileMetadata updates an existing file metadata record
+func (db *DB) UpdateFileMetadata(metadata *FileMetadata) error {
+ return db.Save(metadata).Error
+}
+
+// GetFileMetadataForJob retrieves all file metadata for a job
+func (db *DB) GetFileMetadataForJob(jobID uint) ([]FileMetadata, error) {
+ var metadata []FileMetadata
+ err := db.Where("job_id = ?", jobID).Find(&metadata).Error
+ return metadata, err
+}
+
+// DeleteFileMetadata deletes file metadata by ID
+func (db *DB) DeleteFileMetadata(id uint) error {
+ return db.Delete(&FileMetadata{}, id).Error
+}
+
+// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
}
+// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
+func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
+ var value bool
+ err := db.Model(&TransferConfig{}).
+ Where("id = ?", configID).
+ Select("skip_processed_files").
+ Scan(&value).Error
+ return value, err
+}
+
func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
configPath := db.GetConfigRclonePath(config)
@@ -345,7 +427,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.SourceKeyFile != "" {
args = append(args, "key_file", config.SourceKeyFile)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -362,11 +444,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -383,7 +465,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -397,7 +479,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -412,11 +494,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceDomain != "" {
args = append(args, "domain", config.SourceDomain)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -431,11 +513,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourcePassiveMode {
args = append(args, "passive", "true")
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -450,7 +532,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -466,7 +548,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -480,11 +562,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceDriveID != "" {
args = append(args, "drive_id", config.SourceDriveID)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -498,11 +580,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceTeamDrive != "" {
args = append(args, "team_drive", config.SourceTeamDrive)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -533,7 +615,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.DestKeyFile != "" {
args = append(args, "key_file", config.DestKeyFile)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -550,11 +632,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestEndpoint != "" {
args = append(args, "endpoint", config.DestEndpoint)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -571,7 +653,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -585,7 +667,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -600,11 +682,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestDomain != "" {
args = append(args, "domain", config.DestDomain)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -619,11 +701,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestPassiveMode {
args = append(args, "passive", "true")
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -638,7 +720,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -654,7 +736,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -668,11 +750,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestDriveID != "" {
args = append(args, "drive_id", config.DestDriveID)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -686,11 +768,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestTeamDrive != "" {
args = append(args, "team_drive", config.DestTeamDrive)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
diff --git a/internal/db/migrations/add_skip_processed_files.go b/internal/db/migrations/add_skip_processed_files.go
new file mode 100644
index 0000000..f6939b1
--- /dev/null
+++ b/internal/db/migrations/add_skip_processed_files.go
@@ -0,0 +1,21 @@
+package migrations
+
+import (
+ "github.com/go-gormigrate/gormigrate/v2"
+ "gorm.io/gorm"
+)
+
+// AddSkipProcessedFilesColumn adds the skip_processed_files column to transfer_configs table
+func AddSkipProcessedFilesColumn() *gormigrate.Migration {
+ return &gormigrate.Migration{
+ ID: "20250310_add_skip_processed_files",
+ Migrate: func(tx *gorm.DB) error {
+ // Add skip_processed_files column with default value of true
+ return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN skip_processed_files BOOLEAN DEFAULT true").Error
+ },
+ Rollback: func(tx *gorm.DB) error {
+ // Drop the column if needed
+ return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN skip_processed_files").Error
+ },
+ }
+}
diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go
index c7705a7..49e99a9 100644
--- a/internal/db/migrations/migrations.go
+++ b/internal/db/migrations/migrations.go
@@ -11,7 +11,8 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
// ... existing migrations
AddDeleteAfterTransferColumn(),
AddCloudStorageFields(),
+ AddSkipProcessedFilesColumn(),
}
-
+
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
-}
\ No newline at end of file
+}
diff --git a/internal/email/email.go b/internal/email/email.go
index bcc969d..7d7b9da 100644
--- a/internal/email/email.go
+++ b/internal/email/email.go
@@ -26,12 +26,12 @@ func NewService(cfg *config.Config) *Service {
func (s *Service) SendPasswordResetEmail(toEmail, username, resetToken string) error {
if !s.Config.Email.Enabled {
// If email is not enabled, just log it (you can redirect to the default logging logic)
- return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s",
+ return fmt.Errorf("email service is disabled, reset link would be: %s/reset-password?token=%s",
s.Config.BaseURL, resetToken)
}
resetLink := fmt.Sprintf("%s/reset-password?token=%s", s.Config.BaseURL, resetToken)
-
+
// Create email data for template
data := map[string]interface{}{
"Username": username,
@@ -124,7 +124,7 @@ func (s *Service) generatePasswordResetEmailHTML(data map[string]interface{}) (s
text-align: center;
}
.btn:hover {
- background-color: #4338ca;
+ background-color:rgb(55, 113, 236);
}
.reset-link {
margin: 20px 0;
@@ -210,7 +210,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
headers["Subject"] = subject
headers["MIME-Version"] = "1.0"
headers["Content-Type"] = "text/html; charset=UTF-8"
-
+
if s.Config.Email.ReplyTo != "" {
headers["Reply-To"] = s.Config.Email.ReplyTo
}
@@ -224,7 +224,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
// Set up the SMTP server address
addr := fmt.Sprintf("%s:%d", s.Config.Email.Host, s.Config.Email.Port)
-
+
// Check if authentication is required
if s.Config.Email.RequireAuth {
// Use authenticated SMTP
@@ -237,7 +237,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
return fmt.Errorf("failed to connect to SMTP server: %v", err)
}
defer client.Close()
-
+
// Set up TLS if enabled
if s.Config.Email.EnableTLS {
if err := client.StartTLS(nil); err != nil {
@@ -252,7 +252,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
if err := client.Rcpt(toEmail); err != nil {
return fmt.Errorf("failed to set recipient: %v", err)
}
-
+
// Send the email body
w, err := client.Data()
if err != nil {
@@ -266,7 +266,7 @@ func (s *Service) sendEmail(toEmail, subject, htmlContent string) error {
if err != nil {
return fmt.Errorf("failed to close data writer: %v", err)
}
-
+
return client.Quit()
}
-}
\ No newline at end of file
+}
diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go
index 2f11262..2604ebe 100644
--- a/internal/scheduler/scheduler.go
+++ b/internal/scheduler/scheduler.go
@@ -1,7 +1,11 @@
package scheduler
import (
+ "crypto/md5"
+ "encoding/hex"
+ "encoding/json"
"fmt"
+ "io"
"io/ioutil"
"os"
"os/exec"
@@ -124,12 +128,23 @@ func (s *Scheduler) executeJob(jobID uint) {
return
}
- fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s\n",
+ // Add explicit database reload of the config to ensure we have the latest values
+ var config db.TransferConfig
+ if err := s.db.First(&config, job.Config.ID).Error; err != nil {
+ fmt.Printf("Error loading config %d: %v\n", job.Config.ID, err)
+ return
+ }
+ // Replace the job's config with the freshly loaded one
+ job.Config = config
+
+ // Now the rest of your code will use the correct value
+ fmt.Printf("Loaded job %d with config: source=%s:%s, dest=%s:%s, skipProcessedFiles=%v\n",
jobID,
job.Config.SourceType,
job.Config.SourcePath,
job.Config.DestinationType,
job.Config.DestinationPath,
+ job.Config.SkipProcessedFiles,
)
// Create job history entry
@@ -153,114 +168,227 @@ func (s *Scheduler) executeJob(jobID uint) {
fmt.Printf("Error updating job last run time for job %d: %v\n", jobID, err)
}
+ // Reload the job from the database to get the latest values
+ if err := s.db.Preload("Config").First(&job, jobID).Error; err != nil {
+ fmt.Printf("Error reloading job %d: %v\n", jobID, err)
+ return
+ }
+
+ // Track files already processed in this job execution to prevent duplicates
+ processedFiles := make(map[string]bool)
+
// Get rclone config path
configPath := s.db.GetConfigRclonePath(&job.Config)
- // Size of transfer using rclone size
- sizeArgs := []string{
+ // Use lsjson to get file list and metadata in one operation instead of separate size and ls commands
+ listArgs := []string{
"--config", configPath,
- "size",
- "--include", job.Config.FilePattern,
+ "lsjson",
+ "--hash",
+ "--recursive",
}
-
+
+ // Add file pattern filter if specified
+ if job.Config.FilePattern != "" && job.Config.FilePattern != "*" {
+ // Create a temporary filter file for complex patterns
+ filterFile, err := createRcloneFilterFile(job.Config.FilePattern)
+ if err != nil {
+ fmt.Printf("Error creating filter file for job %d: %v\n", jobID, err)
+ history.Status = "failed"
+ history.ErrorMessage = fmt.Sprintf("Filter Creation Error: %v", err)
+ endTime := time.Now()
+ history.EndTime = &endTime
+ if err := s.db.UpdateJobHistory(history); err != nil {
+ fmt.Printf("Error updating job history for job %d: %v\n", jobID, err)
+ }
+ return
+ }
+ defer os.Remove(filterFile)
+ listArgs = append(listArgs, "--filter-from", filterFile)
+ }
+
// Add source path with bucket for S3-compatible storage
- var sourceSizePath string
+ var sourceListPath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
- sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourceBucket)
+ sourceListPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourceBucket)
if job.Config.SourcePath != "" && job.Config.SourcePath != "/" {
- sourceSizePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath)
+ sourceListPath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath)
}
} else {
- sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
+ sourceListPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
}
-
- sizeArgs = append(sizeArgs, sourceSizePath)
-
- // Get the rclone path from the environment variable or use the default path
+
+ listArgs = append(listArgs, sourceListPath)
+
+ // Execute lsjson command
+ fmt.Printf("Listing files with metadata for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
rclonePath = "rclone"
}
- output, err := exec.Command(rclonePath, sizeArgs...).CombinedOutput()
- fmt.Printf("Running rclone size: %s %s\nOutput: %s\n", rclonePath, strings.Join(sizeArgs, " "), output)
- if err != nil {
- fmt.Printf("Error running rclone size: %v\nOutput: %s\n", err, output)
- // Update job history with error
+ listCmd := exec.Command(rclonePath, listArgs...)
+ listOutput, listErr := listCmd.CombinedOutput()
+
+ if listErr != nil {
+ fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
history.Status = "failed"
- history.ErrorMessage = fmt.Sprintf("Size calculation error: %v\nOutput: %s", err, string(output))
- history.EndTime = &startTime // Use start time as end time for a quick failure
+ history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
+ endTime := time.Now()
+ history.EndTime = &endTime
if err := s.db.UpdateJobHistory(history); err != nil {
fmt.Printf("Error updating job history for job %d: %v\n", jobID, err)
}
return
}
- // Parse rclone size output "Total objects: 1 Total size: 10 B (10 Byte)"
- outputStr := string(output)
+ // Parse JSON output to get file information
+ var fileEntries []map[string]interface{}
+ if err := json.Unmarshal(listOutput, &fileEntries); err != nil {
+ fmt.Printf("Error parsing file list JSON for job %d: %v\n", jobID, err)
+ history.Status = "failed"
+ history.ErrorMessage = fmt.Sprintf("JSON Parsing Error: %v", err)
+ endTime := time.Now()
+ history.EndTime = &endTime
+ if err := s.db.UpdateJobHistory(history); err != nil {
+ fmt.Printf("Error updating job history for job %d: %v\n", jobID, err)
+ }
+ return
+ }
- totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
- totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
- // totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
+ // Calculate total size and filter out directories
+ var files []map[string]interface{}
+ var totalSize int64
+ for _, entry := range fileEntries {
+ // Skip directories
+ if isDir, ok := entry["IsDir"].(bool); ok && isDir {
+ continue
+ }
- if totalObjects == "0" {
+ // Add to files list
+ files = append(files, entry)
+
+ // Add to total size
+ if size, ok := entry["Size"].(float64); ok {
+ totalSize += int64(size)
+ }
+ }
+
+ fmt.Printf("Found %d files totaling %d bytes to transfer for job %d\n", len(files), totalSize, jobID)
+
+ // Update history with size information
+ history.BytesTransferred = totalSize
+
+ if len(files) == 0 {
fmt.Printf("No files to transfer for job %d\n", jobID)
history.Status = "completed"
history.ErrorMessage = ""
history.FilesTransferred = 0
- }
-
- if totalObjects != "0" {
- // First, list all files that match the pattern
- listArgs := []string{
- "--config", configPath,
- "lsf",
- "--include", job.Config.FilePattern,
- }
-
- // Add source path with bucket for S3-compatible storage
- var sourceLsPath string
- if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
- sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourceBucket)
- if job.Config.SourcePath != "" && job.Config.SourcePath != "/" {
- sourceLsPath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath)
- }
- } else {
- sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
- }
-
- listArgs = append(listArgs, sourceLsPath)
-
- fmt.Printf("Listing files for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
- // Get the rclone path from the environment variable or use the default path
- rclonePath := os.Getenv("RCLONE_PATH")
- if rclonePath == "" {
- rclonePath = "rclone"
- }
- listCmd := exec.Command(rclonePath, listArgs...)
- listOutput, listErr := listCmd.CombinedOutput()
-
- if listErr != nil {
- fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
- history.Status = "failed"
- history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
- return
- }
-
- // Split the output by newlines to get individual files
- files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
- fmt.Printf("Found %d files to transfer for job %d\n", len(files), jobID)
-
+ } else {
var transferErrors []string
filesTransferred := 0
-
+
// Process each file individually
- for _, file := range files {
- if file == "" {
+ for _, fileEntry := range files {
+ fileName, ok := fileEntry["Path"].(string)
+ if !ok || fileName == "" {
continue
}
-
- fmt.Printf("Processing file: %s for job %d\n", file, jobID)
-
+
+ // Skip files that have already been processed in this execution
+ if processedFiles[fileName] {
+ fmt.Printf("Skipping duplicate file entry: %s (already processed in this execution)\n", fileName)
+ continue
+ }
+
+ // Extract file metadata from the JSON entry
+ var fileSize int64
+ if size, ok := fileEntry["Size"].(float64); ok {
+ fileSize = int64(size)
+ }
+
+ // Extract modification time
+ modTime := time.Now()
+ if modTimeStr, ok := fileEntry["ModTime"].(string); ok {
+ if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
+ modTime = parsedTime
+ }
+ }
+
+ // Create time is usually not available for remote files, so we'll use modTime
+ createTime := modTime
+
+ // Extract hash if available
+ var fileHash string
+ if hashes, ok := fileEntry["Hashes"].(map[string]interface{}); ok {
+ if md5, ok := hashes["md5"].(string); ok {
+ fileHash = md5
+ }
+ }
+
+ // For local files, calculate hash if not available
+ if fileHash == "" && job.Config.SourceType == "local" {
+ localFilePath := filepath.Join(job.Config.SourcePath, fileName)
+ calculatedHash, hashErr := calculateFileHash(localFilePath)
+ if hashErr == nil {
+ fileHash = calculatedHash
+ }
+ }
+
+ skipFiles := job.Config.SkipProcessedFiles
+
+ // Check if this file has been processed before (by hash)
+ if fileHash != "" {
+ processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+ if processed {
+ fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
+ fileName, fileHash, prevMetadata.FileName)
+
+ // Determine if we should skip this file
+ shouldSkip := false
+ if skipFiles {
+ if prevMetadata.Status == "processed" ||
+ prevMetadata.Status == "archived" ||
+ prevMetadata.Status == "deleted" ||
+ prevMetadata.Status == "archived_and_deleted" {
+ shouldSkip = true
+ }
+ }
+
+ if shouldSkip {
+ fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName)
+ continue
+ } else {
+ fmt.Printf("Re-processing file %s despite previous processing (skipProcessedFiles=%v)\n", fileName, skipFiles)
+ }
+ }
+ }
+
+ // Also check the processing history for this specific file name
+ prevMetadata, histErr := s.checkFileProcessingHistory(jobID, fileName)
+ if histErr == nil {
+ fmt.Printf("File %s was previously processed on %s with status: %s\n",
+ fileName, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+
+ // Determine if we should skip this file based on name+hash match
+ shouldSkip := false
+ if skipFiles && fileHash != "" && fileHash == prevMetadata.FileHash {
+ if prevMetadata.Status == "processed" ||
+ prevMetadata.Status == "archived" ||
+ prevMetadata.Status == "deleted" ||
+ prevMetadata.Status == "archived_and_deleted" {
+ shouldSkip = true
+ }
+ }
+
+ if shouldSkip {
+ fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", fileName)
+ // Skip this file and continue to the next one
+ continue
+ } else if fileHash != "" && fileHash == prevMetadata.FileHash {
+ fmt.Printf("Re-processing file %s despite matching hash (skipProcessedFiles=%v)\n", fileName, skipFiles)
+ }
+ }
+
// Prepare moveto command for transfer
transferArgs := []string{
"--config", configPath,
@@ -270,50 +398,61 @@ func (s *Scheduler) executeJob(jobID uint) {
"--verbose",
"--stats", "1s",
}
-
+
// Source and destination paths
var sourcePath, destPath string
-
+
// For S3, MinIO, and B2, include the bucket in the path
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
- sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, file)
+ sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, fileName)
if job.Config.SourcePath != "" && job.Config.SourcePath != "/" {
- sourcePath = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath, file)
+ sourcePath = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath, fileName)
}
} else {
- sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
+ sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, fileName)
}
-
+
+ var destFile string = fileName
+
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
- destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, fileName)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
- destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, file)
+ destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, fileName)
}
} else {
- destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, file)
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, fileName)
}
-
+
// Add output filename pattern if specified
if job.Config.OutputPattern != "" {
// Process the output pattern for this specific file
- newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
- destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
- fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
+ destFile = ProcessOutputPattern(job.Config.OutputPattern, fileName)
+
+ if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
+ if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+ destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
+ }
+ } else {
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
+ }
+
+ fmt.Printf("Renaming file from %s to %s for job %d\n", fileName, destFile, jobID)
}
-
+
// Add custom flags if specified
if job.Config.RcloneFlags != "" {
customFlags := strings.Split(job.Config.RcloneFlags, " ")
transferArgs = append(transferArgs, customFlags...)
fmt.Printf("Added custom flags for job %d: %v\n", jobID, customFlags)
}
-
+
// Add source and destination to the command
transferArgs = append(transferArgs, sourcePath, destPath)
-
+
// Execute transfer for this file
- fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
- jobID, file, strings.Join(transferArgs, " "))
+ fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
+ jobID, fileName, strings.Join(transferArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -323,39 +462,62 @@ func (s *Scheduler) executeJob(jobID uint) {
fileOutput, fileErr := cmd.CombinedOutput()
// Print the output
- fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
+ fmt.Printf("Output for file %s: %s\n", fileName, string(fileOutput))
+
+ // Create file metadata record
+ fileStatus := "processed"
+ var fileErrorMsg string
+ var destPathForDB string
// Check if file was successfully transferred
if fileErr != nil {
- fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
- transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
+ fmt.Printf("Error transferring file %s for job %d: %v\n", fileName, jobID, fileErr)
+ transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", fileName, fileErr))
+ fileStatus = "error"
+ fileErrorMsg = fileErr.Error()
} else {
filesTransferred++
- fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
-
+ fmt.Printf("Successfully transferred file %s for job %d\n", fileName, jobID)
+
+ // Extract the actual destination path (without rclone remote prefix)
+ if job.Config.DestinationType == "local" {
+ destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
+ } else {
+ // For remote destinations, store the path format
+ if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+ if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+ destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
+ } else {
+ destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
+ }
+ } else {
+ destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
+ }
+ }
+
// If archiving is enabled and transfer was successful, move files to archive
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
- fmt.Printf("Archiving file %s for job %d\n", file, jobID)
-
+ fmt.Printf("Archiving file %s for job %d\n", fileName, jobID)
+
// We don't need to move the file since we used moveto, but we can copy it to archive
archiveArgs := []string{
"--config", configPath,
"copyto",
sourcePath,
}
-
+
// Construct archive path with bucket if needed
var archiveDest string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
- archiveDest = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.ArchivePath, file)
+ archiveDest = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.ArchivePath, fileName)
} else {
- archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, file)
+ archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, fileName)
}
-
+
archiveArgs = append(archiveArgs, archiveDest)
-
- fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
- jobID, file, strings.Join(archiveArgs, " "))
+
+ fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
+ jobID, fileName, strings.Join(archiveArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -365,39 +527,72 @@ func (s *Scheduler) executeJob(jobID uint) {
archiveOutput, archiveErr := archiveCmd.CombinedOutput()
// Print the output
- fmt.Printf("Output for file %s: %s\n", file, string(archiveOutput))
+ fmt.Printf("Output for file %s: %s\n", fileName, string(archiveOutput))
// Check if file was successfully transferred
if archiveErr != nil {
- fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
- transferErrors = append(transferErrors,
- fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
+ fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", fileName, jobID, archiveErr)
+ transferErrors = append(transferErrors,
+ fmt.Sprintf("Archive error for file %s: %v", fileName, archiveErr))
+ } else {
+ fileStatus = "archived"
}
}
+
if job.Config.DeleteAfterTransfer {
- fmt.Printf("Deleting file %s for job %d\n", file, jobID)
+ fmt.Printf("Deleting file %s for job %d\n", fileName, jobID)
deleteArgs := []string{
"--config", configPath,
"deletefile",
- sourcePath, }
+ sourcePath}
deleteCmd := exec.Command(rclonePath, deleteArgs...)
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
- fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
+ fmt.Printf("Output for file %s: %s\n", fileName, string(deleteOutput))
if deleteErr != nil {
- fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
- transferErrors = append(transferErrors,
- fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
+ fmt.Printf("Error deleting file %s for job %d: %v\n", fileName, jobID, deleteErr)
+ transferErrors = append(transferErrors,
+ fmt.Sprintf("Delete error for file %s: %v", fileName, deleteErr))
+ } else {
+ if fileStatus == "archived" {
+ fileStatus = "archived_and_deleted"
+ } else {
+ fileStatus = "deleted"
+ }
}
}
}
+
+ // Mark this file as processed for this execution
+ processedFiles[fileName] = true
+
+ // Create and save file metadata
+ metadata := &db.FileMetadata{
+ JobID: jobID,
+ FileName: fileName,
+ OriginalPath: job.Config.SourcePath,
+ FileSize: fileSize,
+ FileHash: fileHash,
+ CreationTime: createTime,
+ ModTime: modTime,
+ ProcessedTime: time.Now(),
+ DestinationPath: destPathForDB,
+ Status: fileStatus,
+ ErrorMessage: fileErrorMsg,
+ }
+
+ if err := s.db.CreateFileMetadata(metadata); err != nil {
+ fmt.Printf("Error creating file metadata for %s: %v\n", fileName, err)
+ } else {
+ fmt.Printf("Created file metadata record for %s (ID: %d)\n", fileName, metadata.ID)
+ }
}
-
+
// Update job history with transfer results
history.FilesTransferred = filesTransferred
-
+
if len(transferErrors) > 0 {
history.Status = "completed_with_errors"
- history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
+ history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
len(transferErrors), strings.Join(transferErrors, "\n"))
}
}
@@ -439,15 +634,15 @@ func ProcessOutputPattern(pattern string, originalFilename string) string {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
-
+
// Split the filename and extension
ext := filepath.Ext(originalFilename)
filename := strings.TrimSuffix(originalFilename, ext)
-
+
// Replace filename and extension variables
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
-
+
return processedPattern
}
@@ -467,31 +662,31 @@ func createRcloneFilterFile(pattern string) (string, error) {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
-
+
// Replace filename and extension variables with rclone's capture group references
// For rclone rename filters, we need to use {1} for the first capture group, not $1
// See: https://rclone.org/filtering/#rename
-
+
// Extract filename without extension
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
-
+
// Extract extension (with the dot)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
-
+
// Create a rename rule for rclone using the correct syntax:
// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
// - For files with extension: capture the name and extension separately
rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
-
+
// Add a fallback rule for files without extension
- fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
+ fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
strings.ReplaceAll(processedPattern, "{2}", ""))
-
+
// Write the rules to the file
if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
return "", fmt.Errorf("failed to write to filter file: %v", err)
}
-
+
return tmpFile.Name(), nil
}
@@ -515,3 +710,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
go s.executeJob(jobID)
return nil
}
+
+// calculateFileHash computes an MD5 hash for the given file path
+func calculateFileHash(filePath string) (string, error) {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return "", fmt.Errorf("error opening file: %v", err)
+ }
+ defer file.Close()
+
+ hash := md5.New()
+ if _, err := io.Copy(hash, file); err != nil {
+ return "", fmt.Errorf("error calculating hash: %v", err)
+ }
+
+ return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// getFileInfo retrieves file stats like size, creation time, and modification time
+func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
+ info, err := os.Stat(filePath)
+ if err != nil {
+ return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
+ }
+
+ size := info.Size()
+ modTime := info.ModTime()
+
+ // Get creation time (this is platform-specific)
+ // For simplicity, we'll use modification time as a fallback
+ createTime := modTime
+
+ return size, createTime, modTime, nil
+}
+
+// hasFileBeenProcessed checks if a file with the same hash has been processed before
+func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
+ if fileHash == "" {
+ return false, nil, nil
+ }
+
+ // First try to find by hash (most reliable)
+ metadata, err := s.db.GetFileMetadataByHash(fileHash)
+ if err == nil && metadata != nil {
+ return true, metadata, nil
+ }
+
+ return false, nil, nil
+}
+
+// checkFileProcessingHistory checks processing history for a given file
+func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
+ // Try to find by job and filename
+ metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
+ if err == nil && metadata != nil {
+ return metadata, nil
+ }
+
+ return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
+}
+
+// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
+func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
+ // Get rclone config path
+ configPath := s.db.GetConfigRclonePath(config)
+
+ // Construct the appropriate source path
+ var sourcePath string
+ if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
+ sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
+ if config.SourcePath != "" && config.SourcePath != "/" {
+ sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
+ }
+ } else {
+ sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
+ }
+
+ // Use rclone lsjson to get file details
+ rclonePath := os.Getenv("RCLONE_PATH")
+ if rclonePath == "" {
+ rclonePath = "rclone"
+ }
+
+ // Construct the full path to the file
+ fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
+
+ // Run rclone lsjson command
+ args := []string{
+ "--config", configPath,
+ "lsjson",
+ "--hash",
+ fullPath,
+ }
+
+ cmd := exec.Command(rclonePath, args...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
+ }
+
+ // Parse the JSON output
+ var files []map[string]interface{}
+ if err := json.Unmarshal(output, &files); err != nil {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
+ }
+
+ if len(files) == 0 {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
+ }
+
+ fileInfo := files[0]
+
+ // Extract file size
+ var fileSize int64
+ if size, ok := fileInfo["Size"].(float64); ok {
+ fileSize = int64(size)
+ }
+
+ // Extract modification time
+ modTime := time.Now()
+ if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
+ if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
+ modTime = parsedTime
+ }
+ }
+
+ // Create time is usually not available for remote files, so we'll use modTime
+ createTime := modTime
+
+ // Calculate hash if available
+ var md5Hash string
+ if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
+ if md5, ok := hashes["md5"].(string); ok {
+ md5Hash = md5
+ }
+ }
+
+ return fileSize, createTime, modTime, md5Hash, nil
+}
diff --git a/internal/web/handlers/file_metadata_handlers.go b/internal/web/handlers/file_metadata_handlers.go
new file mode 100644
index 0000000..ac35802
--- /dev/null
+++ b/internal/web/handlers/file_metadata_handlers.go
@@ -0,0 +1,645 @@
+package handlers
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "github.com/starfleetcptn/gomft/components"
+ "github.com/starfleetcptn/gomft/internal/db"
+)
+
+// FileMetadataHandler handles displaying and searching file metadata
+type FileMetadataHandler struct {
+ DB *db.DB
+}
+
+// Register registers the file metadata routes
+func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
+ fileGroup := router.Group("/files")
+
+ fileGroup.GET("", h.ListFileMetadata)
+ fileGroup.GET("/:id", h.GetFileMetadataDetails)
+ fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
+ fileGroup.GET("/search", h.SearchFileMetadata)
+ fileGroup.GET("/search/partial", h.HandleFileMetadataSearchPartial)
+ fileGroup.DELETE("/:id", h.DeleteFileMetadata)
+ fileGroup.GET("/partial", h.HandleFileMetadataPartial)
+}
+
+// ListFileMetadata displays a list of file metadata with pagination and filtering options
+func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Query parameters for pagination and filtering
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err := query.Preload("Job").Preload("Job.Config").
+ Order("file_metadata.processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata list template
+ data := components.FileMetadataListData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ // Check if this is an HTMX request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true" || c.Query("htmx") == "true"
+
+ c.Header("Content-Type", "text/html")
+
+ if isHtmxRequest {
+ // For HTMX requests, render just the partial template
+ components.FileMetadataListPartial(data).Render(ctx, c.Writer)
+ } else {
+ // For full page requests, render the complete template
+ components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+ }
+}
+
+// GetFileMetadataDetails displays detailed information about a specific file
+func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Get file ID from URL parameter
+ fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+ return
+ }
+
+ // Retrieve file metadata
+ var fileMetadata db.FileMetadata
+ err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+ return
+ }
+
+ // Check if the user has access to this file (file must belong to a job created by the user)
+ var jobCreator uint
+ err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+ if err != nil || jobCreator != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata details template
+ data := components.FileMetadataDetailsData{
+ File: fileMetadata,
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataForJob displays file metadata for a specific job
+func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Get job ID from URL parameter
+ jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
+ return
+ }
+
+ // Check if the user has access to this job
+ var job db.Job
+ err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
+ return
+ }
+
+ if job.CreatedBy != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
+ return
+ }
+
+ // Query parameters for pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ fileName := c.Query("filename")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
+
+ // Apply filters
+ if status != "" {
+ query = query.Where("status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err = query.Preload("Job").Preload("Job.Config").
+ Order("processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata list template
+ data := components.FileMetadataListData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Job: &job,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: strconv.FormatUint(uint64(job.ID), 10),
+ FileName: fileName,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ // Check if this is an HTMX request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true" || c.Query("htmx") == "true"
+
+ c.Header("Content-Type", "text/html")
+
+ if isHtmxRequest {
+ // For HTMX requests, render just the partial template
+ components.FileMetadataListPartial(data).Render(ctx, c.Writer)
+ } else {
+ // For full page requests, render the complete template
+ components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+ }
+}
+
+// SearchFileMetadata searches file metadata based on various criteria
+func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Query parameters for search and pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+ hash := c.Query("hash")
+ startDate := c.Query("start_date")
+ endDate := c.Query("end_date")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ if hash != "" {
+ query = query.Where("file_metadata.file_hash = ?", hash)
+ }
+
+ if startDate != "" {
+ query = query.Where("file_metadata.processed_time >= ?", startDate)
+ }
+
+ if endDate != "" {
+ query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err := query.Preload("Job").Preload("Job.Config").
+ Order("file_metadata.processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata search template
+ data := components.FileMetadataSearchData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ Hash: hash,
+ StartDate: startDate,
+ EndDate: endDate,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ // Add HTMX request checking and conditional rendering
+ ctx = context.WithValue(c.Request.Context(), "userID", userID)
+
+ // Check if this is an HTMX request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true" || c.Query("htmx") == "true"
+
+ c.Header("Content-Type", "text/html")
+
+ if isHtmxRequest {
+ // For HTMX requests, render just the partial template
+ components.FileMetadataSearchContent(data).Render(ctx, c.Writer)
+ } else {
+ // For full page requests, render the complete template
+ components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
+ }
+}
+
+// DeleteFileMetadata deletes a file metadata record
+func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ fmt.Println("Deleting file metadata")
+
+ // Get file ID from URL parameter
+ fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+ return
+ }
+
+ // Check if the user has access to this file
+ var fileMetadata db.FileMetadata
+ err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+ return
+ }
+
+ var jobCreator uint
+ err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+ if err != nil || jobCreator != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
+ return
+ }
+
+ // Delete the file metadata
+ err = h.DB.DeleteFileMetadata(uint(fileID))
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
+ return
+ }
+
+ fmt.Println("File deleted successfully")
+
+ // Check if this is an HTMX request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true"
+
+ if isHtmxRequest {
+ // For HTMX requests, just return a 200 status - client will handle UI updates
+ c.Status(http.StatusOK)
+ } else {
+ // For regular browser requests, redirect to the file list
+ c.Redirect(http.StatusFound, "/files")
+ }
+}
+
+// HandleFileMetadataPartial handles rendering just the partial template for file metadata
+func (h *FileMetadataHandler) HandleFileMetadataPartial(c *gin.Context) {
+ // Check if this is an HTMX request or a direct browser request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true"
+
+ // If it's a direct browser request (not from HTMX), redirect to the full page
+ if !isHtmxRequest {
+ // Get all query parameters
+ query := c.Request.URL.Query()
+
+ // Rebuild query string for the redirect
+ c.Redirect(http.StatusFound, "/files?"+query.Encode())
+ return
+ }
+
+ userID := c.GetUint("userID")
+
+ // Query parameters for filtering and pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err := query.Preload("Job").Preload("Job.Config").
+ Order("file_metadata.processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Prepare job pointer if needed
+ var job *db.Job
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ var jobRecord db.Job
+ if err := h.DB.DB.First(&jobRecord, jobID).Error; err == nil {
+ job = &jobRecord
+ }
+ }
+
+ // Render the file metadata list template
+ data := components.FileMetadataListData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Job: job,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataListPartial(data).Render(ctx, c.Writer)
+}
+
+// HandleFileMetadataSearchPartial handles partial updates for search results
+func (h *FileMetadataHandler) HandleFileMetadataSearchPartial(c *gin.Context) {
+ // Check if this is an HTMX request or a direct browser request
+ isHtmxRequest := c.GetHeader("HX-Request") == "true"
+
+ // If it's a direct browser request (not from HTMX), redirect to the full page
+ if !isHtmxRequest {
+ // Get all query parameters
+ query := c.Request.URL.Query()
+
+ // Rebuild query string for the redirect
+ c.Redirect(http.StatusFound, "/files/search?"+query.Encode())
+ return
+ }
+
+ userID := c.GetUint("userID")
+
+ // Query parameters for search and pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+ hash := c.Query("hash")
+ startDate := c.Query("start_date")
+ endDate := c.Query("end_date")
+
+ // Execute the search query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ if hash != "" {
+ query = query.Where("file_metadata.file_hash = ?", hash)
+ }
+
+ if startDate != "" {
+ query = query.Where("file_metadata.processed_time >= ?", startDate)
+ }
+
+ if endDate != "" {
+ query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
+ }
+
+ // Count total results
+ var totalCount int64
+ if err := query.Count(&totalCount).Error; err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to count files"})
+ return
+ }
+
+ // Order and paginate the results
+ var files []db.FileMetadata
+ if err := query.
+ Preload("Job").
+ Order("file_metadata.processed_time DESC").
+ Limit(limit).
+ Offset((page - 1) * limit).
+ Find(&files).Error; err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to search files"})
+ return
+ }
+
+ // Build the template data
+ data := components.FileMetadataSearchData{
+ Files: files,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ Hash: hash,
+ StartDate: startDate,
+ EndDate: endDate,
+ },
+ }
+
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ ctx := context.WithValue(c.Request.Context(), "userID", userID)
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataSearchContent(data).Render(ctx, c.Writer)
+}
diff --git a/internal/web/handlers/routes.go b/internal/web/handlers/routes.go
index 20a4244..5c0d086 100644
--- a/internal/web/handlers/routes.go
+++ b/internal/web/handlers/routes.go
@@ -4,8 +4,8 @@ import (
"fmt"
"math"
"net/http"
- "strconv"
"net/url"
+ "strconv"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/components"
@@ -15,13 +15,13 @@ import (
// HandleHistory handles the GET /history route
func (h *Handlers) HandleHistory(c *gin.Context) {
userID := c.GetUint("userID")
-
+
// Get pagination parameters
page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
if err != nil || page < 1 {
page = 1
}
-
+
pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
if err != nil {
pageSize = 10
@@ -30,47 +30,47 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
if pageSize != 10 && pageSize != 25 && pageSize != 50 && pageSize != 100 {
pageSize = 10
}
-
+
// Get search term
searchTerm := c.Query("search")
-
+
// Build the query
query := h.DB.Model(&db.JobHistory{}).
Joins("JOIN jobs ON jobs.id = job_histories.job_id").
Joins("JOIN transfer_configs ON transfer_configs.id = jobs.config_id").
Where("jobs.created_by = ?", userID)
-
+
// Apply search if provided
if searchTerm != "" {
- query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
+ query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
"%"+searchTerm+"%", "%"+searchTerm+"%")
}
-
+
// Count total matching records for pagination
var total int64
query.Count(&total)
-
+
// Calculate total pages
totalPages := int(math.Ceil(float64(total) / float64(pageSize)))
if totalPages == 0 {
totalPages = 1
}
-
+
// Ensure page is within bounds
if page > totalPages {
page = totalPages
}
-
+
// Get paginated results
var history []db.JobHistory
offset := (page - 1) * pageSize
-
+
query.Offset(offset).
Limit(pageSize).
Preload("Job.Config").
Order("start_time desc").
Find(&history)
-
+
// If we got no results and we're not on page 1, redirect to page 1
// Only do this for non-HTMX requests to avoid navigation issues
isHtmxRequest := c.GetHeader("HX-Request") == "true"
@@ -82,7 +82,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
c.Redirect(http.StatusFound, redirectURL)
return
}
-
+
data := components.HistoryData{
History: history,
CurrentPage: page,
@@ -160,10 +160,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
// Protected routes
authorized := router.Group("/")
authorized.Use(h.AuthMiddleware())
-
+
// Password change route - only accessed from profile page
authorized.POST("/change-password", h.HandleChangePassword)
-
+
{
authorized.GET("/dashboard", h.HandleDashboard)
authorized.GET("/configs", h.HandleConfigs)
@@ -186,12 +186,16 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
authorized.GET("/profile", h.HandleProfile)
authorized.POST("/profile/theme", h.HandleUpdateTheme)
authorized.POST("/logout", h.HandleLogout)
-
+
+ // File metadata routes
+ fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
+ fileMetadataHandler.Register(authorized)
+
// AJAX routes for dashboard
authorized.GET("/dashboard/data", h.HandleDashboardData)
authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
authorized.GET("/dashboard/history", h.HandleDashboardHistoryData)
-
+
// Test connection routes
authorized.POST("/test-connection", h.HandleTestConnection)
authorized.POST("/test-sftp-connection", h.HandleTestSFTPConnection)
@@ -208,7 +212,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/users/:id", h.HandleDeleteUser)
admin.GET("/register", h.HandleRegisterPage)
admin.POST("/register", h.HandleRegister)
-
+
// Admin tools routes
admin.GET("/tools", h.HandleAdminTools)
admin.POST("/backup-database", h.HandleBackupDatabase)
@@ -222,12 +226,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/delete-backup/:filename", h.HandleDeleteBackup)
admin.GET("/refresh-backups", h.HandleRefreshBackups)
}
-
+
// API routes
api := router.Group("/api")
{
api.POST("/login", h.HandleAPILogin)
-
+
// Protected API routes
apiAuthorized := api.Group("/")
apiAuthorized.Use(h.APIAuthMiddleware())
@@ -238,7 +242,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.POST("/configs", h.HandleAPICreateConfig)
apiAuthorized.PUT("/configs/:id", h.HandleAPIUpdateConfig)
apiAuthorized.DELETE("/configs/:id", h.HandleAPIDeleteConfig)
-
+
// Job endpoints
apiAuthorized.GET("/jobs", h.HandleAPIJobs)
apiAuthorized.GET("/jobs/:id", h.HandleAPIJob)
@@ -246,11 +250,11 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.PUT("/jobs/:id", h.HandleAPIUpdateJob)
apiAuthorized.DELETE("/jobs/:id", h.HandleAPIDeleteJob)
apiAuthorized.POST("/jobs/:id/run", h.HandleAPIRunJob)
-
+
// History endpoints
apiAuthorized.GET("/history", h.HandleAPIHistory)
apiAuthorized.GET("/job-runs/:id", h.HandleAPIJobRun)
-
+
// Admin-only API routes
apiAdmin := apiAuthorized.Group("/admin")
apiAdmin.Use(h.APIAdminMiddleware())
diff --git a/internal/web/middleware/auth.go b/internal/web/middleware/auth.go
index 0d28fab..87d2cec 100644
--- a/internal/web/middleware/auth.go
+++ b/internal/web/middleware/auth.go
@@ -1,3 +1,4 @@
+
// AuthMiddleware is a middleware function that checks if the request has a valid JWT token
func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
@@ -41,4 +42,4 @@ func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
c.Next()
}
-}
\ No newline at end of file
+}
diff --git a/screenshots/file.metadata.gomft.png b/screenshots/file.metadata.gomft.png
new file mode 100644
index 0000000..129092d
Binary files /dev/null and b/screenshots/file.metadata.gomft.png differ