-
+
+
Archive & Delete Options
+
+
+
-
-
-
Archive files after successful transfer
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-
+
+ Files will be moved here after successful transfer
+
+
+
+
+
+
+ Warning: This will permanently delete the original files
+
-
-
-
-
-
-
-
-
Delete source files after successful transfer
-
-
-
}
templ RcloneFlags() {
diff --git a/components/providers/destination/ftp.templ b/components/providers/destination/ftp.templ
index b9ca774..391b7bb 100644
--- a/components/providers/destination/ftp.templ
+++ b/components/providers/destination/ftp.templ
@@ -94,6 +94,7 @@ templ FTPDestinationForm() {
name="dest_passive_mode"
type="checkbox"
x-model="destPassiveMode"
+ :value="destPassiveMode ? 'true' : 'false'"
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
diff --git a/components/providers/source/ftp.templ b/components/providers/source/ftp.templ
index 8ec116a..201ba59 100644
--- a/components/providers/source/ftp.templ
+++ b/components/providers/source/ftp.templ
@@ -94,6 +94,7 @@ templ FTPSourceForm() {
name="source_passive_mode"
type="checkbox"
x-model="sourcePassiveMode"
+ :value="sourcePassiveMode ? 'true' : 'false'"
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
diff --git a/internal/db/db.go b/internal/db/db.go
index 1779f94..7a6a12d 100644
--- a/internal/db/db.go
+++ b/internal/db/db.go
@@ -7,23 +7,23 @@ import (
"path/filepath"
"time"
- "github.com/starfleetcptn/gomft/internal/auth"
"github.com/glebarez/sqlite"
+ "github.com/starfleetcptn/gomft/internal/auth"
"gorm.io/gorm"
)
type User struct {
- ID uint `gorm:"primarykey"`
- Email string `gorm:"unique;not null"`
- PasswordHash string `gorm:"not null"`
- IsAdmin bool `gorm:"default:false"`
- LastPasswordChange time.Time
- FailedLoginAttempts int `gorm:"default:0"`
- AccountLocked bool `gorm:"default:false"`
- LockoutUntil *time.Time
- Theme string `gorm:"default:'light'"`
- CreatedAt time.Time
- UpdatedAt time.Time
+ ID uint `gorm:"primarykey"`
+ Email string `gorm:"unique;not null"`
+ PasswordHash string `gorm:"not null"`
+ IsAdmin bool `gorm:"default:false"`
+ LastPasswordChange time.Time
+ FailedLoginAttempts int `gorm:"default:0"`
+ AccountLocked bool `gorm:"default:false"`
+ LockoutUntil *time.Time
+ Theme string `gorm:"default:'light'"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
}
type PasswordHistory struct {
@@ -46,31 +46,31 @@ type PasswordResetToken struct {
}
type TransferConfig struct {
- ID uint `gorm:"primarykey"`
- Name string `gorm:"not null" form:"name"`
- SourceType string `gorm:"not null" form:"source_type"`
- SourcePath string `gorm:"not null" form:"source_path"`
- SourceHost string `form:"source_host"`
- SourcePort int `gorm:"default:22" form:"source_port"`
- SourceUser string `form:"source_user"`
- SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
- SourceKeyFile string `form:"source_key_file"`
+ ID uint `gorm:"primarykey"`
+ Name string `gorm:"not null" form:"name"`
+ SourceType string `gorm:"not null" form:"source_type"`
+ SourcePath string `gorm:"not null" form:"source_path"`
+ SourceHost string `form:"source_host"`
+ SourcePort int `gorm:"default:22" form:"source_port"`
+ SourceUser string `form:"source_user"`
+ SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
+ SourceKeyFile string `form:"source_key_file"`
// S3 source fields
- SourceBucket string `form:"source_bucket"`
- SourceRegion string `form:"source_region"`
- SourceAccessKey string `form:"source_access_key"`
- SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
- SourceEndpoint string `form:"source_endpoint"`
+ SourceBucket string `form:"source_bucket"`
+ SourceRegion string `form:"source_region"`
+ SourceAccessKey string `form:"source_access_key"`
+ SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
+ SourceEndpoint string `form:"source_endpoint"`
// SMB source fields
- SourceShare string `form:"source_share"`
- SourceDomain string `form:"source_domain"`
+ SourceShare string `form:"source_share"`
+ SourceDomain string `form:"source_domain"`
// FTP source fields
- SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
+ SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
// OneDrive and Google Drive source fields
SourceClientID string `form:"source_client_id"`
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
- SourceDriveID string `form:"source_drive_id"` // For OneDrive
- SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
+ SourceDriveID string `form:"source_drive_id"` // For OneDrive
+ SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
// General fields
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
@@ -82,30 +82,30 @@ type TransferConfig struct {
DestPassword string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
DestKeyFile string `form:"dest_key_file"`
// S3 destination fields
- DestBucket string `form:"dest_bucket"`
- DestRegion string `form:"dest_region"`
- DestAccessKey string `form:"dest_access_key"`
- DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
- DestEndpoint string `form:"dest_endpoint"`
+ DestBucket string `form:"dest_bucket"`
+ DestRegion string `form:"dest_region"`
+ DestAccessKey string `form:"dest_access_key"`
+ DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
+ DestEndpoint string `form:"dest_endpoint"`
// SMB destination fields
- DestShare string `form:"dest_share"`
- DestDomain string `form:"dest_domain"`
+ DestShare string `form:"dest_share"`
+ DestDomain string `form:"dest_domain"`
// FTP destination fields
- DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
+ DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
// OneDrive and Google Drive destination fields
DestClientID string `form:"dest_client_id"`
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
- DestDriveID string `form:"dest_drive_id"` // For OneDrive
- DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
+ DestDriveID string `form:"dest_drive_id"` // For OneDrive
+ DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
// General fields
- ArchivePath string `form:"archive_path"`
- ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
- RcloneFlags string `form:"rclone_flags"`
+ ArchivePath string `form:"archive_path"`
+ ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
+ RcloneFlags string `form:"rclone_flags"`
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
- CreatedBy uint
- User User `gorm:"foreignkey:CreatedBy"`
- CreatedAt time.Time
- UpdatedAt time.Time
+ CreatedBy uint
+ User User `gorm:"foreignkey:CreatedBy"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
}
type Job struct {
@@ -135,6 +135,25 @@ type JobHistory struct {
ErrorMessage string
}
+// FileMetadata stores information about processed files
+type FileMetadata struct {
+ ID uint `gorm:"primarykey"`
+ JobID uint `gorm:"not null;index"`
+ Job Job `gorm:"foreignkey:JobID"`
+ FileName string `gorm:"not null"`
+ OriginalPath string `gorm:"not null"`
+ FileSize int64 `gorm:"not null"`
+ FileHash string `gorm:"index"` // MD5 or other hash for file identity
+ CreationTime time.Time
+ ModTime time.Time
+ ProcessedTime time.Time `gorm:"not null"`
+ DestinationPath string `gorm:"not null"`
+ Status string `gorm:"not null"` // processed, archived, deleted, etc.
+ ErrorMessage string
+ CreatedAt time.Time
+ UpdatedAt time.Time
+}
+
type DB struct {
*gorm.DB
}
@@ -153,7 +172,7 @@ func Initialize(dbPath string) (*DB, error) {
}
// Auto migrate the schema
- err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{})
+ err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{}, &FileMetadata{})
if err != nil {
return nil, fmt.Errorf("failed to migrate database: %v", err)
}
@@ -302,12 +321,63 @@ func (db *DB) UpdateJobHistory(history *JobHistory) error {
}
func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
- var history []JobHistory
- err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&history).Error
- return history, err
+ var histories []JobHistory
+ err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&histories).Error
+ return histories, err
+}
+
+// CreateFileMetadata creates a new file metadata record
+func (db *DB) CreateFileMetadata(metadata *FileMetadata) error {
+ return db.Create(metadata).Error
+}
+
+// GetFileMetadata retrieves file metadata by ID
+func (db *DB) GetFileMetadata(id uint) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.First(&metadata, id).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// GetFileMetadataByJobAndName retrieves file metadata by job ID and filename
+func (db *DB) GetFileMetadataByJobAndName(jobID uint, fileName string) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.Where("job_id = ? AND file_name = ?", jobID, fileName).First(&metadata).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// GetFileMetadataByHash retrieves file metadata by file hash
+func (db *DB) GetFileMetadataByHash(fileHash string) (*FileMetadata, error) {
+ var metadata FileMetadata
+ err := db.Where("file_hash = ?", fileHash).First(&metadata).Error
+ if err != nil {
+ return nil, err
+ }
+ return &metadata, nil
+}
+
+// UpdateFileMetadata updates an existing file metadata record
+func (db *DB) UpdateFileMetadata(metadata *FileMetadata) error {
+ return db.Save(metadata).Error
+}
+
+// GetFileMetadataForJob retrieves all file metadata for a job
+func (db *DB) GetFileMetadataForJob(jobID uint) ([]FileMetadata, error) {
+ var metadata []FileMetadata
+ err := db.Where("job_id = ?", jobID).Find(&metadata).Error
+ return metadata, err
+}
+
+// DeleteFileMetadata deletes file metadata by ID
+func (db *DB) DeleteFileMetadata(id uint) error {
+ return db.Delete(&FileMetadata{}, id).Error
}
-// Helper functions
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
}
@@ -345,7 +415,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.SourceKeyFile != "" {
args = append(args, "key_file", config.SourceKeyFile)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -362,11 +432,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -383,7 +453,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -397,7 +467,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -412,11 +482,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceDomain != "" {
args = append(args, "domain", config.SourceDomain)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -431,11 +501,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourcePassiveMode {
args = append(args, "passive", "true")
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -450,7 +520,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -466,7 +536,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -480,11 +550,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceDriveID != "" {
args = append(args, "drive_id", config.SourceDriveID)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -498,11 +568,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.SourceTeamDrive != "" {
args = append(args, "team_drive", config.SourceTeamDrive)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -533,7 +603,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.DestKeyFile != "" {
args = append(args, "key_file", config.DestKeyFile)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -550,11 +620,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestEndpoint != "" {
args = append(args, "endpoint", config.DestEndpoint)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -571,7 +641,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -585,7 +655,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -600,11 +670,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestDomain != "" {
args = append(args, "domain", config.DestDomain)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -619,11 +689,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestPassiveMode {
args = append(args, "passive", "true")
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -638,7 +708,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -654,7 +724,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -668,11 +738,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestDriveID != "" {
args = append(args, "drive_id", config.DestDriveID)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -686,11 +756,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
-
+
if config.DestTeamDrive != "" {
args = append(args, "team_drive", config.DestTeamDrive)
}
-
+
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go
index 2f11262..5ce1fe4 100644
--- a/internal/scheduler/scheduler.go
+++ b/internal/scheduler/scheduler.go
@@ -1,12 +1,17 @@
package scheduler
import (
+ "crypto/md5"
+ "encoding/hex"
+ "encoding/json"
"fmt"
+ "io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
+ "strconv"
"strings"
"sync"
"time"
@@ -162,7 +167,7 @@ func (s *Scheduler) executeJob(jobID uint) {
"size",
"--include", job.Config.FilePattern,
}
-
+
// Add source path with bucket for S3-compatible storage
var sourceSizePath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -173,9 +178,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
}
-
+
sizeArgs = append(sizeArgs, sourceSizePath)
-
+
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -200,7 +205,16 @@ func (s *Scheduler) executeJob(jobID uint) {
totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
- // totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
+ totalSize := strings.TrimSpace(strings.Split(outputStr, "Total size:")[1])
+ if strings.Contains(totalSize, "(") {
+ totalSize = strings.TrimSpace(strings.Split(totalSize, "(")[1])
+ totalSize = strings.TrimSpace(strings.Split(totalSize, " ")[0])
+ } else {
+ totalSize = "0"
+ }
+
+ bytesTransferred, _ := strconv.ParseInt(totalSize, 10, 64)
+ history.BytesTransferred = bytesTransferred
if totalObjects == "0" {
fmt.Printf("No files to transfer for job %d\n", jobID)
@@ -216,7 +230,7 @@ func (s *Scheduler) executeJob(jobID uint) {
"lsf",
"--include", job.Config.FilePattern,
}
-
+
// Add source path with bucket for S3-compatible storage
var sourceLsPath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -227,9 +241,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
}
-
+
listArgs = append(listArgs, sourceLsPath)
-
+
fmt.Printf("Listing files for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -238,29 +252,123 @@ func (s *Scheduler) executeJob(jobID uint) {
}
listCmd := exec.Command(rclonePath, listArgs...)
listOutput, listErr := listCmd.CombinedOutput()
-
+
if listErr != nil {
fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
history.Status = "failed"
history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
return
}
-
+
// Split the output by newlines to get individual files
files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
fmt.Printf("Found %d files to transfer for job %d\n", len(files), jobID)
-
+
var transferErrors []string
filesTransferred := 0
-
+
// Process each file individually
for _, file := range files {
if file == "" {
continue
}
-
+
fmt.Printf("Processing file: %s for job %d\n", file, jobID)
-
+
+ // Get detailed file info if this is a local source
+ var fileSize int64
+ var createTime, modTime time.Time
+ var fileHash string
+ var fileMetadataErr error
+
+ // Get file metadata based on source type
+ if job.Config.SourceType == "local" {
+ // Construct the full local path
+ localFilePath := filepath.Join(job.Config.SourcePath, file)
+
+ // Get file info (size, creation time, modification time)
+ fileSize, createTime, modTime, fileMetadataErr = getFileInfo(localFilePath)
+ if fileMetadataErr != nil {
+ fmt.Printf("Warning: Could not get file info for %s: %v\n", file, fileMetadataErr)
+ }
+
+ // Calculate file hash (MD5)
+ fileHash, fileMetadataErr = calculateFileHash(localFilePath)
+ if fileMetadataErr != nil {
+ fmt.Printf("Warning: Could not calculate hash for %s: %v\n", file, fileMetadataErr)
+ }
+
+ // Check if this file has been processed before (by hash)
+ if fileHash != "" {
+ processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+ if processed {
+ fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
+ file, fileHash, prevMetadata.FileName)
+
+ // If configured to skip previously processed files,
+ // we could add that logic here
+ // For now, we'll just log it and continue
+ }
+ }
+
+ // Also check the processing history for this specific file name
+ prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+ if histErr == nil {
+ fmt.Printf("File %s was previously processed on %s with status: %s\n",
+ file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+
+ // If the file was previously processed successfully and the hash hasn't changed,
+ // we could skip processing
+ if prevMetadata.Status == "processed" ||
+ prevMetadata.Status == "archived" ||
+ prevMetadata.Status == "deleted" ||
+ prevMetadata.Status == "archived_and_deleted" {
+ if fileHash != "" && fileHash == prevMetadata.FileHash {
+ fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", file)
+ // Skip this file and continue to the next one
+ continue
+ }
+ }
+ }
+ } else {
+ // For non-local sources, use rclone lsjson to get metadata
+ fileSize, createTime, modTime, fileHash, fileMetadataErr = s.getRemoteFileInfo(&job.Config, file)
+ if fileMetadataErr != nil {
+ fmt.Printf("Warning: Could not get remote file info for %s: %v\n", file, fileMetadataErr)
+ // We'll continue with placeholder values
+ fileSize = 0
+ createTime = time.Now()
+ modTime = time.Now()
+ fileHash = ""
+ } else {
+ // If we got a hash, check for previous processing
+ if fileHash != "" {
+ processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+ if processed {
+ fmt.Printf("Remote file %s has been processed before (hash: %s, previous file: %s)\n",
+ file, fileHash, prevMetadata.FileName)
+
+ // Skip previously processed files with the same hash if they were processed successfully
+ if prevMetadata.Status == "processed" ||
+ prevMetadata.Status == "archived" ||
+ prevMetadata.Status == "deleted" ||
+ prevMetadata.Status == "archived_and_deleted" {
+ fmt.Printf("Skipping unchanged remote file %s (hash matches previous processing)\n", file)
+ // Skip this file and continue to the next one
+ continue
+ }
+ }
+ }
+
+ // Also check by filename
+ prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+ if histErr == nil {
+ fmt.Printf("Remote file %s was previously processed on %s with status: %s\n",
+ file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+ }
+ }
+ }
+
// Prepare moveto command for transfer
transferArgs := []string{
"--config", configPath,
@@ -270,10 +378,10 @@ func (s *Scheduler) executeJob(jobID uint) {
"--verbose",
"--stats", "1s",
}
-
+
// Source and destination paths
var sourcePath, destPath string
-
+
// For S3, MinIO, and B2, include the bucket in the path
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, file)
@@ -283,7 +391,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
}
-
+
+ var destFile string = file
+
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
@@ -292,27 +402,36 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, file)
}
-
+
// Add output filename pattern if specified
if job.Config.OutputPattern != "" {
// Process the output pattern for this specific file
- newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
- destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
- fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
+ destFile = ProcessOutputPattern(job.Config.OutputPattern, file)
+
+ if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
+ if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+ destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
+ }
+ } else {
+ destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
+ }
+
+ fmt.Printf("Renaming file from %s to %s for job %d\n", file, destFile, jobID)
}
-
+
// Add custom flags if specified
if job.Config.RcloneFlags != "" {
customFlags := strings.Split(job.Config.RcloneFlags, " ")
transferArgs = append(transferArgs, customFlags...)
fmt.Printf("Added custom flags for job %d: %v\n", jobID, customFlags)
}
-
+
// Add source and destination to the command
transferArgs = append(transferArgs, sourcePath, destPath)
-
+
// Execute transfer for this file
- fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
+ fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
jobID, file, strings.Join(transferArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -325,25 +444,48 @@ func (s *Scheduler) executeJob(jobID uint) {
// Print the output
fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
+ // Create file metadata record
+ fileStatus := "processed"
+ var fileErrorMsg string
+ var destPathForDB string
+
// Check if file was successfully transferred
if fileErr != nil {
fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
+ fileStatus = "error"
+ fileErrorMsg = fileErr.Error()
} else {
filesTransferred++
fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
-
+
+ // Extract the actual destination path (without rclone remote prefix)
+ if job.Config.DestinationType == "local" {
+ destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
+ } else {
+ // For remote destinations, store the path format
+ if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+ if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+ destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
+ } else {
+ destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
+ }
+ } else {
+ destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
+ }
+ }
+
// If archiving is enabled and transfer was successful, move files to archive
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
fmt.Printf("Archiving file %s for job %d\n", file, jobID)
-
+
// We don't need to move the file since we used moveto, but we can copy it to archive
archiveArgs := []string{
"--config", configPath,
"copyto",
sourcePath,
}
-
+
// Construct archive path with bucket if needed
var archiveDest string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -351,10 +493,10 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, file)
}
-
+
archiveArgs = append(archiveArgs, archiveDest)
-
- fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
+
+ fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
jobID, file, strings.Join(archiveArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -370,34 +512,64 @@ func (s *Scheduler) executeJob(jobID uint) {
// Check if file was successfully transferred
if archiveErr != nil {
fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
- transferErrors = append(transferErrors,
+ transferErrors = append(transferErrors,
fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
+ } else {
+ fileStatus = "archived"
}
}
+
if job.Config.DeleteAfterTransfer {
fmt.Printf("Deleting file %s for job %d\n", file, jobID)
deleteArgs := []string{
"--config", configPath,
"deletefile",
- sourcePath, }
+ sourcePath}
deleteCmd := exec.Command(rclonePath, deleteArgs...)
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
if deleteErr != nil {
fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
- transferErrors = append(transferErrors,
+ transferErrors = append(transferErrors,
fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
+ } else {
+ if fileStatus == "archived" {
+ fileStatus = "archived_and_deleted"
+ } else {
+ fileStatus = "deleted"
+ }
}
}
}
+
+ // Create and save file metadata
+ metadata := &db.FileMetadata{
+ JobID: jobID,
+ FileName: file,
+ OriginalPath: job.Config.SourcePath,
+ FileSize: fileSize,
+ FileHash: fileHash,
+ CreationTime: createTime,
+ ModTime: modTime,
+ ProcessedTime: time.Now(),
+ DestinationPath: destPathForDB,
+ Status: fileStatus,
+ ErrorMessage: fileErrorMsg,
+ }
+
+ if err := s.db.CreateFileMetadata(metadata); err != nil {
+ fmt.Printf("Error creating file metadata for %s: %v\n", file, err)
+ } else {
+ fmt.Printf("Created file metadata record for %s (ID: %d)\n", file, metadata.ID)
+ }
}
-
+
// Update job history with transfer results
history.FilesTransferred = filesTransferred
-
+
if len(transferErrors) > 0 {
history.Status = "completed_with_errors"
- history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
+ history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
len(transferErrors), strings.Join(transferErrors, "\n"))
}
}
@@ -439,15 +611,15 @@ func ProcessOutputPattern(pattern string, originalFilename string) string {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
-
+
// Split the filename and extension
ext := filepath.Ext(originalFilename)
filename := strings.TrimSuffix(originalFilename, ext)
-
+
// Replace filename and extension variables
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
-
+
return processedPattern
}
@@ -467,31 +639,31 @@ func createRcloneFilterFile(pattern string) (string, error) {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
-
+
// Replace filename and extension variables with rclone's capture group references
// For rclone rename filters, we need to use {1} for the first capture group, not $1
// See: https://rclone.org/filtering/#rename
-
+
// Extract filename without extension
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
-
+
// Extract extension (with the dot)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
-
+
// Create a rename rule for rclone using the correct syntax:
// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
// - For files with extension: capture the name and extension separately
rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
-
+
// Add a fallback rule for files without extension
- fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
+ fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
strings.ReplaceAll(processedPattern, "{2}", ""))
-
+
// Write the rules to the file
if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
return "", fmt.Errorf("failed to write to filter file: %v", err)
}
-
+
return tmpFile.Name(), nil
}
@@ -515,3 +687,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
go s.executeJob(jobID)
return nil
}
+
+// calculateFileHash computes an MD5 hash for the given file path
+func calculateFileHash(filePath string) (string, error) {
+ file, err := os.Open(filePath)
+ if err != nil {
+ return "", fmt.Errorf("error opening file: %v", err)
+ }
+ defer file.Close()
+
+ hash := md5.New()
+ if _, err := io.Copy(hash, file); err != nil {
+ return "", fmt.Errorf("error calculating hash: %v", err)
+ }
+
+ return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// getFileInfo retrieves file stats like size, creation time, and modification time
+func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
+ info, err := os.Stat(filePath)
+ if err != nil {
+ return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
+ }
+
+ size := info.Size()
+ modTime := info.ModTime()
+
+ // Get creation time (this is platform-specific)
+ // For simplicity, we'll use modification time as a fallback
+ createTime := modTime
+
+ return size, createTime, modTime, nil
+}
+
+// hasFileBeenProcessed checks if a file with the same hash has been processed before
+func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
+ if fileHash == "" {
+ return false, nil, nil
+ }
+
+ // First try to find by hash (most reliable)
+ metadata, err := s.db.GetFileMetadataByHash(fileHash)
+ if err == nil && metadata != nil {
+ return true, metadata, nil
+ }
+
+ return false, nil, nil
+}
+
+// checkFileProcessingHistory checks processing history for a given file
+func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
+ // Try to find by job and filename
+ metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
+ if err == nil && metadata != nil {
+ return metadata, nil
+ }
+
+ return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
+}
+
+// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
+func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
+ // Get rclone config path
+ configPath := s.db.GetConfigRclonePath(config)
+
+ // Construct the appropriate source path
+ var sourcePath string
+ if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
+ sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
+ if config.SourcePath != "" && config.SourcePath != "/" {
+ sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
+ }
+ } else {
+ sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
+ }
+
+ // Use rclone lsjson to get file details
+ rclonePath := os.Getenv("RCLONE_PATH")
+ if rclonePath == "" {
+ rclonePath = "rclone"
+ }
+
+ // Construct the full path to the file
+ fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
+
+ // Run rclone lsjson command
+ args := []string{
+ "--config", configPath,
+ "lsjson",
+ "--hash",
+ fullPath,
+ }
+
+ cmd := exec.Command(rclonePath, args...)
+ output, err := cmd.CombinedOutput()
+ if err != nil {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
+ }
+
+ // Parse the JSON output
+ var files []map[string]interface{}
+ if err := json.Unmarshal(output, &files); err != nil {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
+ }
+
+ if len(files) == 0 {
+ return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
+ }
+
+ fileInfo := files[0]
+
+ // Extract file size
+ var fileSize int64
+ if size, ok := fileInfo["Size"].(float64); ok {
+ fileSize = int64(size)
+ }
+
+ // Extract modification time
+ modTime := time.Now()
+ if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
+ if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
+ modTime = parsedTime
+ }
+ }
+
+ // Create time is usually not available for remote files, so we'll use modTime
+ createTime := modTime
+
+ // Calculate hash if available
+ var md5Hash string
+ if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
+ if md5, ok := hashes["md5"].(string); ok {
+ md5Hash = md5
+ }
+ }
+
+ return fileSize, createTime, modTime, md5Hash, nil
+}
diff --git a/internal/web/handlers/file_metadata_handlers.go b/internal/web/handlers/file_metadata_handlers.go
new file mode 100644
index 0000000..09526d8
--- /dev/null
+++ b/internal/web/handlers/file_metadata_handlers.go
@@ -0,0 +1,378 @@
+package handlers
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/gin-gonic/gin"
+ "github.com/starfleetcptn/gomft/components"
+ "github.com/starfleetcptn/gomft/internal/db"
+)
+
+// FileMetadataHandler handles displaying and searching file metadata
+type FileMetadataHandler struct {
+ DB *db.DB
+}
+
+// Register registers the file metadata routes
+func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
+ fileGroup := router.Group("/files")
+
+ fileGroup.GET("", h.ListFileMetadata)
+ fileGroup.GET("/:id", h.GetFileMetadataDetails)
+ fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
+ fileGroup.GET("/search", h.SearchFileMetadata)
+ fileGroup.DELETE("/:id", h.DeleteFileMetadata)
+}
+
+// ListFileMetadata displays a list of file metadata with pagination and filtering options
+func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Query parameters for pagination and filtering
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err := query.Preload("Job").Preload("Job.Config").
+ Order("file_metadata.processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata list template
+ data := components.FileMetadataListData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataDetails displays detailed information about a specific file
+func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Get file ID from URL parameter
+ fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+ return
+ }
+
+ // Retrieve file metadata
+ var fileMetadata db.FileMetadata
+ err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+ return
+ }
+
+ // Check if the user has access to this file (file must belong to a job created by the user)
+ var jobCreator uint
+ err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+ if err != nil || jobCreator != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata details template
+ data := components.FileMetadataDetailsData{
+ File: fileMetadata,
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataForJob displays file metadata for a specific job
+func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Get job ID from URL parameter
+ jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
+ return
+ }
+
+ // Check if the user has access to this job
+ var job db.Job
+ err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
+ return
+ }
+
+ if job.CreatedBy != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
+ return
+ }
+
+ // Query parameters for pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ fileName := c.Query("filename")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
+
+ // Apply filters
+ if status != "" {
+ query = query.Where("status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err = query.Preload("Job").Preload("Job.Config").
+ Order("processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata list template
+ data := components.FileMetadataListData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Job: &job,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: strconv.FormatUint(uint64(job.ID), 10),
+ FileName: fileName,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// SearchFileMetadata searches file metadata based on various criteria
+func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Query parameters for search and pagination
+ page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+ if page < 1 {
+ page = 1
+ }
+
+ limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+ if limit < 1 || limit > 100 {
+ limit = 50
+ }
+
+ status := c.Query("status")
+ jobIDStr := c.Query("job_id")
+ fileName := c.Query("filename")
+ hash := c.Query("hash")
+ startDate := c.Query("start_date")
+ endDate := c.Query("end_date")
+
+ // Base query
+ query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+ // Apply filters
+ if jobIDStr != "" {
+ jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+ query = query.Where("file_metadata.job_id = ?", jobID)
+ } else {
+ // Only show files from jobs created by the current user
+ query = query.Where("jobs.created_by = ?", userID)
+ }
+
+ if status != "" {
+ query = query.Where("file_metadata.status = ?", status)
+ }
+
+ if fileName != "" {
+ query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+ }
+
+ if hash != "" {
+ query = query.Where("file_metadata.file_hash = ?", hash)
+ }
+
+ if startDate != "" {
+ query = query.Where("file_metadata.processed_time >= ?", startDate)
+ }
+
+ if endDate != "" {
+ query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
+ }
+
+ // Count total records for pagination
+ var totalCount int64
+ query.Count(&totalCount)
+
+ // Retrieve file metadata with pagination
+ var fileMetadata []db.FileMetadata
+ offset := (page - 1) * limit
+ err := query.Preload("Job").Preload("Job.Config").
+ Order("file_metadata.processed_time DESC").
+ Offset(offset).Limit(limit).
+ Find(&fileMetadata).Error
+
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+ return
+ }
+
+ // Create context for template
+ ctx := components.CreateTemplateContext(c)
+
+ // Render the file metadata search template
+ data := components.FileMetadataSearchData{
+ Files: fileMetadata,
+ TotalCount: totalCount,
+ Page: page,
+ Limit: limit,
+ TotalPages: int(totalCount) / limit,
+ Filter: components.FileMetadataFilter{
+ Status: status,
+ JobID: jobIDStr,
+ FileName: fileName,
+ Hash: hash,
+ StartDate: startDate,
+ EndDate: endDate,
+ },
+ }
+
+ // If total count is not exactly divisible by limit, add one more page
+ if int(totalCount)%limit > 0 {
+ data.TotalPages++
+ }
+
+ c.Header("Content-Type", "text/html")
+ components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
+}
+
+// DeleteFileMetadata deletes a file metadata record
+func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
+ userID := c.GetUint("userID")
+
+ // Get file ID from URL parameter
+ fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+ if err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+ return
+ }
+
+ // Check if the user has access to this file
+ var fileMetadata db.FileMetadata
+ err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
+ if err != nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+ return
+ }
+
+ var jobCreator uint
+ err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+ if err != nil || jobCreator != userID {
+ c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
+ return
+ }
+
+ // Delete the file metadata
+ err = h.DB.DeleteFileMetadata(uint(fileID))
+ if err != nil {
+ c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
+ return
+ }
+
+ // Redirect to the file list
+ c.Redirect(http.StatusFound, "/files")
+}
diff --git a/internal/web/handlers/routes.go b/internal/web/handlers/routes.go
index 20a4244..5c0d086 100644
--- a/internal/web/handlers/routes.go
+++ b/internal/web/handlers/routes.go
@@ -4,8 +4,8 @@ import (
"fmt"
"math"
"net/http"
- "strconv"
"net/url"
+ "strconv"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/components"
@@ -15,13 +15,13 @@ import (
// HandleHistory handles the GET /history route
func (h *Handlers) HandleHistory(c *gin.Context) {
userID := c.GetUint("userID")
-
+
// Get pagination parameters
page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
if err != nil || page < 1 {
page = 1
}
-
+
pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
if err != nil {
pageSize = 10
@@ -30,47 +30,47 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
if pageSize != 10 && pageSize != 25 && pageSize != 50 && pageSize != 100 {
pageSize = 10
}
-
+
// Get search term
searchTerm := c.Query("search")
-
+
// Build the query
query := h.DB.Model(&db.JobHistory{}).
Joins("JOIN jobs ON jobs.id = job_histories.job_id").
Joins("JOIN transfer_configs ON transfer_configs.id = jobs.config_id").
Where("jobs.created_by = ?", userID)
-
+
// Apply search if provided
if searchTerm != "" {
- query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
+ query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
"%"+searchTerm+"%", "%"+searchTerm+"%")
}
-
+
// Count total matching records for pagination
var total int64
query.Count(&total)
-
+
// Calculate total pages
totalPages := int(math.Ceil(float64(total) / float64(pageSize)))
if totalPages == 0 {
totalPages = 1
}
-
+
// Ensure page is within bounds
if page > totalPages {
page = totalPages
}
-
+
// Get paginated results
var history []db.JobHistory
offset := (page - 1) * pageSize
-
+
query.Offset(offset).
Limit(pageSize).
Preload("Job.Config").
Order("start_time desc").
Find(&history)
-
+
// If we got no results and we're not on page 1, redirect to page 1
// Only do this for non-HTMX requests to avoid navigation issues
isHtmxRequest := c.GetHeader("HX-Request") == "true"
@@ -82,7 +82,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
c.Redirect(http.StatusFound, redirectURL)
return
}
-
+
data := components.HistoryData{
History: history,
CurrentPage: page,
@@ -160,10 +160,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
// Protected routes
authorized := router.Group("/")
authorized.Use(h.AuthMiddleware())
-
+
// Password change route - only accessed from profile page
authorized.POST("/change-password", h.HandleChangePassword)
-
+
{
authorized.GET("/dashboard", h.HandleDashboard)
authorized.GET("/configs", h.HandleConfigs)
@@ -186,12 +186,16 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
authorized.GET("/profile", h.HandleProfile)
authorized.POST("/profile/theme", h.HandleUpdateTheme)
authorized.POST("/logout", h.HandleLogout)
-
+
+ // File metadata routes
+ fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
+ fileMetadataHandler.Register(authorized)
+
// AJAX routes for dashboard
authorized.GET("/dashboard/data", h.HandleDashboardData)
authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
authorized.GET("/dashboard/history", h.HandleDashboardHistoryData)
-
+
// Test connection routes
authorized.POST("/test-connection", h.HandleTestConnection)
authorized.POST("/test-sftp-connection", h.HandleTestSFTPConnection)
@@ -208,7 +212,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/users/:id", h.HandleDeleteUser)
admin.GET("/register", h.HandleRegisterPage)
admin.POST("/register", h.HandleRegister)
-
+
// Admin tools routes
admin.GET("/tools", h.HandleAdminTools)
admin.POST("/backup-database", h.HandleBackupDatabase)
@@ -222,12 +226,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/delete-backup/:filename", h.HandleDeleteBackup)
admin.GET("/refresh-backups", h.HandleRefreshBackups)
}
-
+
// API routes
api := router.Group("/api")
{
api.POST("/login", h.HandleAPILogin)
-
+
// Protected API routes
apiAuthorized := api.Group("/")
apiAuthorized.Use(h.APIAuthMiddleware())
@@ -238,7 +242,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.POST("/configs", h.HandleAPICreateConfig)
apiAuthorized.PUT("/configs/:id", h.HandleAPIUpdateConfig)
apiAuthorized.DELETE("/configs/:id", h.HandleAPIDeleteConfig)
-
+
// Job endpoints
apiAuthorized.GET("/jobs", h.HandleAPIJobs)
apiAuthorized.GET("/jobs/:id", h.HandleAPIJob)
@@ -246,11 +250,11 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.PUT("/jobs/:id", h.HandleAPIUpdateJob)
apiAuthorized.DELETE("/jobs/:id", h.HandleAPIDeleteJob)
apiAuthorized.POST("/jobs/:id/run", h.HandleAPIRunJob)
-
+
// History endpoints
apiAuthorized.GET("/history", h.HandleAPIHistory)
apiAuthorized.GET("/job-runs/:id", h.HandleAPIJobRun)
-
+
// Admin-only API routes
apiAdmin := apiAuthorized.Group("/admin")
apiAdmin.Use(h.APIAdminMiddleware())
diff --git a/internal/web/middleware/auth.go b/internal/web/middleware/auth.go
index 0d28fab..87d2cec 100644
--- a/internal/web/middleware/auth.go
+++ b/internal/web/middleware/auth.go
@@ -1,3 +1,4 @@
+
// AuthMiddleware is a middleware function that checks if the request has a valid JWT token
func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
@@ -41,4 +42,4 @@ func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
c.Next()
}
-}
\ No newline at end of file
+}