Github-Mirror/GoMFT
1
0
Fork 0
mirror of https://github.com/StarFleetCPTN/GoMFT.git synced 2026-09-08 23:50:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: Add file metadata tracking and search functionality

Browse Source 
- Implement FileMetadata model to track processed files
- Create file metadata handlers for listing, searching, and viewing files
- Add file metadata routes and UI components
- Support advanced file search with multiple filters
- Enhance job execution to capture file metadata during transfers
- Implement file hash and duplicate detection logic
This commit is contained in:
StarFleetCPTN
2025-03-09 10:22:57 -07:00
parent 3b9ff3b6a8
commit fcf6ee25a8
10 changed files with 1570 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/db.go
+153 -83
View File
@@ -7,23 +7,23 @@ import (
"path/filepath"
"time"
"github.com/starfleetcptn/gomft/internal/auth"
"github.com/glebarez/sqlite"
"github.com/starfleetcptn/gomft/internal/auth"
"gorm.io/gorm"
)
type User struct {
ID uint `gorm:"primarykey"`
Email string `gorm:"unique;not null"`
PasswordHash string `gorm:"not null"`
IsAdmin bool `gorm:"default:false"`
LastPasswordChange time.Time
FailedLoginAttempts int `gorm:"default:0"`
AccountLocked bool `gorm:"default:false"`
LockoutUntil *time.Time
Theme string `gorm:"default:'light'"`
CreatedAt time.Time
UpdatedAt time.Time
ID uint `gorm:"primarykey"`
Email string `gorm:"unique;not null"`
PasswordHash string `gorm:"not null"`
IsAdmin bool `gorm:"default:false"`
LastPasswordChange time.Time
FailedLoginAttempts int `gorm:"default:0"`
AccountLocked bool `gorm:"default:false"`
LockoutUntil *time.Time
Theme string `gorm:"default:'light'"`
CreatedAt time.Time
UpdatedAt time.Time
}
type PasswordHistory struct {
@@ -46,31 +46,31 @@ type PasswordResetToken struct {
}
type TransferConfig struct {
ID uint `gorm:"primarykey"`
Name string `gorm:"not null" form:"name"`
SourceType string `gorm:"not null" form:"source_type"`
SourcePath string `gorm:"not null" form:"source_path"`
SourceHost string `form:"source_host"`
SourcePort int `gorm:"default:22" form:"source_port"`
SourceUser string `form:"source_user"`
SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
SourceKeyFile string `form:"source_key_file"`
ID uint `gorm:"primarykey"`
Name string `gorm:"not null" form:"name"`
SourceType string `gorm:"not null" form:"source_type"`
SourcePath string `gorm:"not null" form:"source_path"`
SourceHost string `form:"source_host"`
SourcePort int `gorm:"default:22" form:"source_port"`
SourceUser string `form:"source_user"`
SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
SourceKeyFile string `form:"source_key_file"`
// S3 source fields
SourceBucket string `form:"source_bucket"`
SourceRegion string `form:"source_region"`
SourceAccessKey string `form:"source_access_key"`
SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
SourceEndpoint string `form:"source_endpoint"`
SourceBucket string `form:"source_bucket"`
SourceRegion string `form:"source_region"`
SourceAccessKey string `form:"source_access_key"`
SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
SourceEndpoint string `form:"source_endpoint"`
// SMB source fields
SourceShare string `form:"source_share"`
SourceDomain string `form:"source_domain"`
SourceShare string `form:"source_share"`
SourceDomain string `form:"source_domain"`
// FTP source fields
SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
// OneDrive and Google Drive source fields
SourceClientID string `form:"source_client_id"`
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
SourceDriveID string `form:"source_drive_id"` // For OneDrive
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
SourceDriveID string `form:"source_drive_id"` // For OneDrive
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
// General fields
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
@@ -82,30 +82,30 @@ type TransferConfig struct {
DestPassword string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
DestKeyFile string `form:"dest_key_file"`
// S3 destination fields
DestBucket string `form:"dest_bucket"`
DestRegion string `form:"dest_region"`
DestAccessKey string `form:"dest_access_key"`
DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
DestEndpoint string `form:"dest_endpoint"`
DestBucket string `form:"dest_bucket"`
DestRegion string `form:"dest_region"`
DestAccessKey string `form:"dest_access_key"`
DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
DestEndpoint string `form:"dest_endpoint"`
// SMB destination fields
DestShare string `form:"dest_share"`
DestDomain string `form:"dest_domain"`
DestShare string `form:"dest_share"`
DestDomain string `form:"dest_domain"`
// FTP destination fields
DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
// OneDrive and Google Drive destination fields
DestClientID string `form:"dest_client_id"`
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
DestDriveID string `form:"dest_drive_id"` // For OneDrive
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
DestDriveID string `form:"dest_drive_id"` // For OneDrive
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
// General fields
ArchivePath string `form:"archive_path"`
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
RcloneFlags string `form:"rclone_flags"`
ArchivePath string `form:"archive_path"`
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
RcloneFlags string `form:"rclone_flags"`
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
CreatedBy uint
User User `gorm:"foreignkey:CreatedBy"`
CreatedAt time.Time
UpdatedAt time.Time
CreatedBy uint
User User `gorm:"foreignkey:CreatedBy"`
CreatedAt time.Time
UpdatedAt time.Time
}
type Job struct {
@@ -135,6 +135,25 @@ type JobHistory struct {
ErrorMessage string
}
// FileMetadata stores information about processed files
type FileMetadata struct {
ID uint `gorm:"primarykey"`
JobID uint `gorm:"not null;index"`
Job Job `gorm:"foreignkey:JobID"`
FileName string `gorm:"not null"`
OriginalPath string `gorm:"not null"`
FileSize int64 `gorm:"not null"`
FileHash string `gorm:"index"` // MD5 or other hash for file identity
CreationTime time.Time
ModTime time.Time
ProcessedTime time.Time `gorm:"not null"`
DestinationPath string `gorm:"not null"`
Status string `gorm:"not null"` // processed, archived, deleted, etc.
ErrorMessage string
CreatedAt time.Time
UpdatedAt time.Time
}
type DB struct {
*gorm.DB
}
@@ -153,7 +172,7 @@ func Initialize(dbPath string) (*DB, error) {
}
// Auto migrate the schema
err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{})
err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{}, &FileMetadata{})
if err != nil {
return nil, fmt.Errorf("failed to migrate database: %v", err)
}
@@ -302,12 +321,63 @@ func (db *DB) UpdateJobHistory(history *JobHistory) error {
}
func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
var history []JobHistory
err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&history).Error
return history, err
var histories []JobHistory
err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&histories).Error
return histories, err
}
// CreateFileMetadata creates a new file metadata record
func (db *DB) CreateFileMetadata(metadata *FileMetadata) error {
return db.Create(metadata).Error
}
// GetFileMetadata retrieves file metadata by ID
func (db *DB) GetFileMetadata(id uint) (*FileMetadata, error) {
var metadata FileMetadata
err := db.First(&metadata, id).Error
if err != nil {
return nil, err
}
return &metadata, nil
}
// GetFileMetadataByJobAndName retrieves file metadata by job ID and filename
func (db *DB) GetFileMetadataByJobAndName(jobID uint, fileName string) (*FileMetadata, error) {
var metadata FileMetadata
err := db.Where("job_id = ? AND file_name = ?", jobID, fileName).First(&metadata).Error
if err != nil {
return nil, err
}
return &metadata, nil
}
// GetFileMetadataByHash retrieves file metadata by file hash
func (db *DB) GetFileMetadataByHash(fileHash string) (*FileMetadata, error) {
var metadata FileMetadata
err := db.Where("file_hash = ?", fileHash).First(&metadata).Error
if err != nil {
return nil, err
}
return &metadata, nil
}
// UpdateFileMetadata updates an existing file metadata record
func (db *DB) UpdateFileMetadata(metadata *FileMetadata) error {
return db.Save(metadata).Error
}
// GetFileMetadataForJob retrieves all file metadata for a job
func (db *DB) GetFileMetadataForJob(jobID uint) ([]FileMetadata, error) {
var metadata []FileMetadata
err := db.Where("job_id = ?", jobID).Find(&metadata).Error
return metadata, err
}
// DeleteFileMetadata deletes file metadata by ID
func (db *DB) DeleteFileMetadata(id uint) error {
return db.Delete(&FileMetadata{}, id).Error
}
// Helper functions
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
}
@@ -345,7 +415,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.SourceKeyFile != "" {
args = append(args, "key_file", config.SourceKeyFile)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -362,11 +432,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -383,7 +453,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -397,7 +467,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -412,11 +482,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceDomain != "" {
args = append(args, "domain", config.SourceDomain)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -431,11 +501,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourcePassiveMode {
args = append(args, "passive", "true")
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -450,7 +520,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -466,7 +536,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -480,11 +550,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceDriveID != "" {
args = append(args, "drive_id", config.SourceDriveID)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -498,11 +568,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceTeamDrive != "" {
args = append(args, "team_drive", config.SourceTeamDrive)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -533,7 +603,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
if config.DestKeyFile != "" {
args = append(args, "key_file", config.DestKeyFile)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -550,11 +620,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestEndpoint != "" {
args = append(args, "endpoint", config.DestEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -571,7 +641,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -585,7 +655,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -600,11 +670,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestDomain != "" {
args = append(args, "domain", config.DestDomain)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -619,11 +689,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestPassiveMode {
args = append(args, "passive", "true")
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -638,7 +708,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -654,7 +724,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -668,11 +738,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestDriveID != "" {
args = append(args, "drive_id", config.DestDriveID)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -686,11 +756,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestTeamDrive != "" {
args = append(args, "team_drive", config.DestTeamDrive)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
internal/scheduler/scheduler.go
+357 -47
View File
@@ -1,12 +1,17 @@
package scheduler
import (
"crypto/md5"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"
"time"
@@ -162,7 +167,7 @@ func (s *Scheduler) executeJob(jobID uint) {
"size",
"--include", job.Config.FilePattern,
}
// Add source path with bucket for S3-compatible storage
var sourceSizePath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -173,9 +178,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
}
sizeArgs = append(sizeArgs, sourceSizePath)
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -200,7 +205,16 @@ func (s *Scheduler) executeJob(jobID uint) {
totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
// totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
totalSize := strings.TrimSpace(strings.Split(outputStr, "Total size:")[1])
if strings.Contains(totalSize, "(") {
totalSize = strings.TrimSpace(strings.Split(totalSize, "(")[1])
totalSize = strings.TrimSpace(strings.Split(totalSize, " ")[0])
} else {
totalSize = "0"
}
bytesTransferred, _ := strconv.ParseInt(totalSize, 10, 64)
history.BytesTransferred = bytesTransferred
if totalObjects == "0" {
fmt.Printf("No files to transfer for job %d\n", jobID)
@@ -216,7 +230,7 @@ func (s *Scheduler) executeJob(jobID uint) {
"lsf",
"--include", job.Config.FilePattern,
}
// Add source path with bucket for S3-compatible storage
var sourceLsPath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -227,9 +241,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
}
listArgs = append(listArgs, sourceLsPath)
fmt.Printf("Listing files for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -238,29 +252,123 @@ func (s *Scheduler) executeJob(jobID uint) {
}
listCmd := exec.Command(rclonePath, listArgs...)
listOutput, listErr := listCmd.CombinedOutput()
if listErr != nil {
fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
history.Status = "failed"
history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
return
}
// Split the output by newlines to get individual files
files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
fmt.Printf("Found %d files to transfer for job %d\n", len(files), jobID)
var transferErrors []string
filesTransferred := 0
// Process each file individually
for _, file := range files {
if file == "" {
continue
}
fmt.Printf("Processing file: %s for job %d\n", file, jobID)
// Get detailed file info if this is a local source
var fileSize int64
var createTime, modTime time.Time
var fileHash string
var fileMetadataErr error
// Get file metadata based on source type
if job.Config.SourceType == "local" {
// Construct the full local path
localFilePath := filepath.Join(job.Config.SourcePath, file)
// Get file info (size, creation time, modification time)
fileSize, createTime, modTime, fileMetadataErr = getFileInfo(localFilePath)
if fileMetadataErr != nil {
fmt.Printf("Warning: Could not get file info for %s: %v\n", file, fileMetadataErr)
}
// Calculate file hash (MD5)
fileHash, fileMetadataErr = calculateFileHash(localFilePath)
if fileMetadataErr != nil {
fmt.Printf("Warning: Could not calculate hash for %s: %v\n", file, fileMetadataErr)
}
// Check if this file has been processed before (by hash)
if fileHash != "" {
processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
if processed {
fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
file, fileHash, prevMetadata.FileName)
// If configured to skip previously processed files,
// we could add that logic here
// For now, we'll just log it and continue
}
}
// Also check the processing history for this specific file name
prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
if histErr == nil {
fmt.Printf("File %s was previously processed on %s with status: %s\n",
file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
// If the file was previously processed successfully and the hash hasn't changed,
// we could skip processing
if prevMetadata.Status == "processed" ||
prevMetadata.Status == "archived" ||
prevMetadata.Status == "deleted" ||
prevMetadata.Status == "archived_and_deleted" {
if fileHash != "" && fileHash == prevMetadata.FileHash {
fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", file)
// Skip this file and continue to the next one
continue
}
}
}
} else {
// For non-local sources, use rclone lsjson to get metadata
fileSize, createTime, modTime, fileHash, fileMetadataErr = s.getRemoteFileInfo(&job.Config, file)
if fileMetadataErr != nil {
fmt.Printf("Warning: Could not get remote file info for %s: %v\n", file, fileMetadataErr)
// We'll continue with placeholder values
fileSize = 0
createTime = time.Now()
modTime = time.Now()
fileHash = ""
} else {
// If we got a hash, check for previous processing
if fileHash != "" {
processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
if processed {
fmt.Printf("Remote file %s has been processed before (hash: %s, previous file: %s)\n",
file, fileHash, prevMetadata.FileName)
// Skip previously processed files with the same hash if they were processed successfully
if prevMetadata.Status == "processed" ||
prevMetadata.Status == "archived" ||
prevMetadata.Status == "deleted" ||
prevMetadata.Status == "archived_and_deleted" {
fmt.Printf("Skipping unchanged remote file %s (hash matches previous processing)\n", file)
// Skip this file and continue to the next one
continue
}
}
}
// Also check by filename
prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
if histErr == nil {
fmt.Printf("Remote file %s was previously processed on %s with status: %s\n",
file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
}
}
}
// Prepare moveto command for transfer
transferArgs := []string{
"--config", configPath,
@@ -270,10 +378,10 @@ func (s *Scheduler) executeJob(jobID uint) {
"--verbose",
"--stats", "1s",
}
// Source and destination paths
var sourcePath, destPath string
// For S3, MinIO, and B2, include the bucket in the path
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, file)
@@ -283,7 +391,9 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
}
var destFile string = file
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
@@ -292,27 +402,36 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, file)
}
// Add output filename pattern if specified
if job.Config.OutputPattern != "" {
// Process the output pattern for this specific file
newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
destFile = ProcessOutputPattern(job.Config.OutputPattern, file)
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
}
} else {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
}
fmt.Printf("Renaming file from %s to %s for job %d\n", file, destFile, jobID)
}
// Add custom flags if specified
if job.Config.RcloneFlags != "" {
customFlags := strings.Split(job.Config.RcloneFlags, " ")
transferArgs = append(transferArgs, customFlags...)
fmt.Printf("Added custom flags for job %d: %v\n", jobID, customFlags)
}
// Add source and destination to the command
transferArgs = append(transferArgs, sourcePath, destPath)
// Execute transfer for this file
fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
jobID, file, strings.Join(transferArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -325,25 +444,48 @@ func (s *Scheduler) executeJob(jobID uint) {
// Print the output
fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
// Create file metadata record
fileStatus := "processed"
var fileErrorMsg string
var destPathForDB string
// Check if file was successfully transferred
if fileErr != nil {
fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
fileStatus = "error"
fileErrorMsg = fileErr.Error()
} else {
filesTransferred++
fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
// Extract the actual destination path (without rclone remote prefix)
if job.Config.DestinationType == "local" {
destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
} else {
// For remote destinations, store the path format
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
} else {
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
}
} else {
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
}
}
// If archiving is enabled and transfer was successful, move files to archive
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
fmt.Printf("Archiving file %s for job %d\n", file, jobID)
// We don't need to move the file since we used moveto, but we can copy it to archive
archiveArgs := []string{
"--config", configPath,
"copyto",
sourcePath,
}
// Construct archive path with bucket if needed
var archiveDest string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -351,10 +493,10 @@ func (s *Scheduler) executeJob(jobID uint) {
} else {
archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, file)
}
archiveArgs = append(archiveArgs, archiveDest)
fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
jobID, file, strings.Join(archiveArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
@@ -370,34 +512,64 @@ func (s *Scheduler) executeJob(jobID uint) {
// Check if file was successfully transferred
if archiveErr != nil {
fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
transferErrors = append(transferErrors,
transferErrors = append(transferErrors,
fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
} else {
fileStatus = "archived"
}
}
if job.Config.DeleteAfterTransfer {
fmt.Printf("Deleting file %s for job %d\n", file, jobID)
deleteArgs := []string{
"--config", configPath,
"deletefile",
sourcePath, }
sourcePath}
deleteCmd := exec.Command(rclonePath, deleteArgs...)
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
if deleteErr != nil {
fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
transferErrors = append(transferErrors,
transferErrors = append(transferErrors,
fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
} else {
if fileStatus == "archived" {
fileStatus = "archived_and_deleted"
} else {
fileStatus = "deleted"
}
}
}
}
// Create and save file metadata
metadata := &db.FileMetadata{
JobID: jobID,
FileName: file,
OriginalPath: job.Config.SourcePath,
FileSize: fileSize,
FileHash: fileHash,
CreationTime: createTime,
ModTime: modTime,
ProcessedTime: time.Now(),
DestinationPath: destPathForDB,
Status: fileStatus,
ErrorMessage: fileErrorMsg,
}
if err := s.db.CreateFileMetadata(metadata); err != nil {
fmt.Printf("Error creating file metadata for %s: %v\n", file, err)
} else {
fmt.Printf("Created file metadata record for %s (ID: %d)\n", file, metadata.ID)
}
}
// Update job history with transfer results
history.FilesTransferred = filesTransferred
if len(transferErrors) > 0 {
history.Status = "completed_with_errors"
history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
len(transferErrors), strings.Join(transferErrors, "\n"))
}
}
@@ -439,15 +611,15 @@ func ProcessOutputPattern(pattern string, originalFilename string) string {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
// Split the filename and extension
ext := filepath.Ext(originalFilename)
filename := strings.TrimSuffix(originalFilename, ext)
// Replace filename and extension variables
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
return processedPattern
}
@@ -467,31 +639,31 @@ func createRcloneFilterFile(pattern string) (string, error) {
format := dateRegex.FindStringSubmatch(match)[1]
return time.Now().Format(format)
})
// Replace filename and extension variables with rclone's capture group references
// For rclone rename filters, we need to use {1} for the first capture group, not $1
// See: https://rclone.org/filtering/#rename
// Extract filename without extension
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
// Extract extension (with the dot)
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
// Create a rename rule for rclone using the correct syntax:
// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
// - For files with extension: capture the name and extension separately
rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
// Add a fallback rule for files without extension
fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
strings.ReplaceAll(processedPattern, "{2}", ""))
// Write the rules to the file
if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
return "", fmt.Errorf("failed to write to filter file: %v", err)
}
return tmpFile.Name(), nil
}
@@ -515,3 +687,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
go s.executeJob(jobID)
return nil
}
// calculateFileHash computes an MD5 hash for the given file path
func calculateFileHash(filePath string) (string, error) {
file, err := os.Open(filePath)
if err != nil {
return "", fmt.Errorf("error opening file: %v", err)
}
defer file.Close()
hash := md5.New()
if _, err := io.Copy(hash, file); err != nil {
return "", fmt.Errorf("error calculating hash: %v", err)
}
return hex.EncodeToString(hash.Sum(nil)), nil
}
// getFileInfo retrieves file stats like size, creation time, and modification time
func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
info, err := os.Stat(filePath)
if err != nil {
return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
}
size := info.Size()
modTime := info.ModTime()
// Get creation time (this is platform-specific)
// For simplicity, we'll use modification time as a fallback
createTime := modTime
return size, createTime, modTime, nil
}
// hasFileBeenProcessed checks if a file with the same hash has been processed before
func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
if fileHash == "" {
return false, nil, nil
}
// First try to find by hash (most reliable)
metadata, err := s.db.GetFileMetadataByHash(fileHash)
if err == nil && metadata != nil {
return true, metadata, nil
}
return false, nil, nil
}
// checkFileProcessingHistory checks processing history for a given file
func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
// Try to find by job and filename
metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
if err == nil && metadata != nil {
return metadata, nil
}
return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
}
// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
// Get rclone config path
configPath := s.db.GetConfigRclonePath(config)
// Construct the appropriate source path
var sourcePath string
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
if config.SourcePath != "" && config.SourcePath != "/" {
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
}
} else {
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
}
// Use rclone lsjson to get file details
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
rclonePath = "rclone"
}
// Construct the full path to the file
fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
// Run rclone lsjson command
args := []string{
"--config", configPath,
"lsjson",
"--hash",
fullPath,
}
cmd := exec.Command(rclonePath, args...)
output, err := cmd.CombinedOutput()
if err != nil {
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
}
// Parse the JSON output
var files []map[string]interface{}
if err := json.Unmarshal(output, &files); err != nil {
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
}
if len(files) == 0 {
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
}
fileInfo := files[0]
// Extract file size
var fileSize int64
if size, ok := fileInfo["Size"].(float64); ok {
fileSize = int64(size)
}
// Extract modification time
modTime := time.Now()
if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
modTime = parsedTime
}
}
// Create time is usually not available for remote files, so we'll use modTime
createTime := modTime
// Calculate hash if available
var md5Hash string
if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
if md5, ok := hashes["md5"].(string); ok {
md5Hash = md5
}
}
return fileSize, createTime, modTime, md5Hash, nil
}
internal/web/handlers/file_metadata_handlers.go
+378
View File
@@ -0,0 +1,378 @@
package handlers
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/components"
"github.com/starfleetcptn/gomft/internal/db"
)
// FileMetadataHandler handles displaying and searching file metadata
type FileMetadataHandler struct {
DB *db.DB
}
// Register registers the file metadata routes
func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
fileGroup := router.Group("/files")
fileGroup.GET("", h.ListFileMetadata)
fileGroup.GET("/:id", h.GetFileMetadataDetails)
fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
fileGroup.GET("/search", h.SearchFileMetadata)
fileGroup.DELETE("/:id", h.DeleteFileMetadata)
}
// ListFileMetadata displays a list of file metadata with pagination and filtering options
func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
userID := c.GetUint("userID")
// Query parameters for pagination and filtering
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
if page < 1 {
page = 1
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
if limit < 1 || limit > 100 {
limit = 50
}
status := c.Query("status")
jobIDStr := c.Query("job_id")
fileName := c.Query("filename")
// Base query
query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
// Apply filters
if jobIDStr != "" {
jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
query = query.Where("file_metadata.job_id = ?", jobID)
} else {
// Only show files from jobs created by the current user
query = query.Where("jobs.created_by = ?", userID)
}
if status != "" {
query = query.Where("file_metadata.status = ?", status)
}
if fileName != "" {
query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
}
// Count total records for pagination
var totalCount int64
query.Count(&totalCount)
// Retrieve file metadata with pagination
var fileMetadata []db.FileMetadata
offset := (page - 1) * limit
err := query.Preload("Job").Preload("Job.Config").
Order("file_metadata.processed_time DESC").
Offset(offset).Limit(limit).
Find(&fileMetadata).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
return
}
// Create context for template
ctx := components.CreateTemplateContext(c)
// Render the file metadata list template
data := components.FileMetadataListData{
Files: fileMetadata,
TotalCount: totalCount,
Page: page,
Limit: limit,
TotalPages: int(totalCount) / limit,
Filter: components.FileMetadataFilter{
Status: status,
JobID: jobIDStr,
FileName: fileName,
},
}
// If total count is not exactly divisible by limit, add one more page
if int(totalCount)%limit > 0 {
data.TotalPages++
}
c.Header("Content-Type", "text/html")
components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
}
// GetFileMetadataDetails displays detailed information about a specific file
func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
userID := c.GetUint("userID")
// Get file ID from URL parameter
fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
return
}
// Retrieve file metadata
var fileMetadata db.FileMetadata
err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
return
}
// Check if the user has access to this file (file must belong to a job created by the user)
var jobCreator uint
err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
if err != nil || jobCreator != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
return
}
// Create context for template
ctx := components.CreateTemplateContext(c)
// Render the file metadata details template
data := components.FileMetadataDetailsData{
File: fileMetadata,
}
c.Header("Content-Type", "text/html")
components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
}
// GetFileMetadataForJob displays file metadata for a specific job
func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
userID := c.GetUint("userID")
// Get job ID from URL parameter
jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
return
}
// Check if the user has access to this job
var job db.Job
err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
return
}
if job.CreatedBy != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
return
}
// Query parameters for pagination
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
if page < 1 {
page = 1
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
if limit < 1 || limit > 100 {
limit = 50
}
status := c.Query("status")
fileName := c.Query("filename")
// Base query
query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
// Apply filters
if status != "" {
query = query.Where("status = ?", status)
}
if fileName != "" {
query = query.Where("file_name LIKE ?", "%"+fileName+"%")
}
// Count total records for pagination
var totalCount int64
query.Count(&totalCount)
// Retrieve file metadata with pagination
var fileMetadata []db.FileMetadata
offset := (page - 1) * limit
err = query.Preload("Job").Preload("Job.Config").
Order("processed_time DESC").
Offset(offset).Limit(limit).
Find(&fileMetadata).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
return
}
// Create context for template
ctx := components.CreateTemplateContext(c)
// Render the file metadata list template
data := components.FileMetadataListData{
Files: fileMetadata,
TotalCount: totalCount,
Page: page,
Limit: limit,
TotalPages: int(totalCount) / limit,
Job: &job,
Filter: components.FileMetadataFilter{
Status: status,
JobID: strconv.FormatUint(uint64(job.ID), 10),
FileName: fileName,
},
}
// If total count is not exactly divisible by limit, add one more page
if int(totalCount)%limit > 0 {
data.TotalPages++
}
c.Header("Content-Type", "text/html")
components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
}
// SearchFileMetadata searches file metadata based on various criteria
func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
userID := c.GetUint("userID")
// Query parameters for search and pagination
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
if page < 1 {
page = 1
}
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
if limit < 1 || limit > 100 {
limit = 50
}
status := c.Query("status")
jobIDStr := c.Query("job_id")
fileName := c.Query("filename")
hash := c.Query("hash")
startDate := c.Query("start_date")
endDate := c.Query("end_date")
// Base query
query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
// Apply filters
if jobIDStr != "" {
jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
query = query.Where("file_metadata.job_id = ?", jobID)
} else {
// Only show files from jobs created by the current user
query = query.Where("jobs.created_by = ?", userID)
}
if status != "" {
query = query.Where("file_metadata.status = ?", status)
}
if fileName != "" {
query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
}
if hash != "" {
query = query.Where("file_metadata.file_hash = ?", hash)
}
if startDate != "" {
query = query.Where("file_metadata.processed_time >= ?", startDate)
}
if endDate != "" {
query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
}
// Count total records for pagination
var totalCount int64
query.Count(&totalCount)
// Retrieve file metadata with pagination
var fileMetadata []db.FileMetadata
offset := (page - 1) * limit
err := query.Preload("Job").Preload("Job.Config").
Order("file_metadata.processed_time DESC").
Offset(offset).Limit(limit).
Find(&fileMetadata).Error
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
return
}
// Create context for template
ctx := components.CreateTemplateContext(c)
// Render the file metadata search template
data := components.FileMetadataSearchData{
Files: fileMetadata,
TotalCount: totalCount,
Page: page,
Limit: limit,
TotalPages: int(totalCount) / limit,
Filter: components.FileMetadataFilter{
Status: status,
JobID: jobIDStr,
FileName: fileName,
Hash: hash,
StartDate: startDate,
EndDate: endDate,
},
}
// If total count is not exactly divisible by limit, add one more page
if int(totalCount)%limit > 0 {
data.TotalPages++
}
c.Header("Content-Type", "text/html")
components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
}
// DeleteFileMetadata deletes a file metadata record
func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
userID := c.GetUint("userID")
// Get file ID from URL parameter
fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
if err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
return
}
// Check if the user has access to this file
var fileMetadata db.FileMetadata
err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
if err != nil {
c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
return
}
var jobCreator uint
err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
if err != nil || jobCreator != userID {
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
return
}
// Delete the file metadata
err = h.DB.DeleteFileMetadata(uint(fileID))
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
return
}
// Redirect to the file list
c.Redirect(http.StatusFound, "/files")
}
internal/web/handlers/routes.go
+28 -24
View File
@@ -4,8 +4,8 @@ import (
"fmt"
"math"
"net/http"
"strconv"
"net/url"
"strconv"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/components"
@@ -15,13 +15,13 @@ import (
// HandleHistory handles the GET /history route
func (h *Handlers) HandleHistory(c *gin.Context) {
userID := c.GetUint("userID")
// Get pagination parameters
page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
if err != nil || page < 1 {
page = 1
}
pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
if err != nil {
pageSize = 10
@@ -30,47 +30,47 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
if pageSize != 10 && pageSize != 25 && pageSize != 50 && pageSize != 100 {
pageSize = 10
}
// Get search term
searchTerm := c.Query("search")
// Build the query
query := h.DB.Model(&db.JobHistory{}).
Joins("JOIN jobs ON jobs.id = job_histories.job_id").
Joins("JOIN transfer_configs ON transfer_configs.id = jobs.config_id").
Where("jobs.created_by = ?", userID)
// Apply search if provided
if searchTerm != "" {
query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
"%"+searchTerm+"%", "%"+searchTerm+"%")
}
// Count total matching records for pagination
var total int64
query.Count(&total)
// Calculate total pages
totalPages := int(math.Ceil(float64(total) / float64(pageSize)))
if totalPages == 0 {
totalPages = 1
}
// Ensure page is within bounds
if page > totalPages {
page = totalPages
}
// Get paginated results
var history []db.JobHistory
offset := (page - 1) * pageSize
query.Offset(offset).
Limit(pageSize).
Preload("Job.Config").
Order("start_time desc").
Find(&history)
// If we got no results and we're not on page 1, redirect to page 1
// Only do this for non-HTMX requests to avoid navigation issues
isHtmxRequest := c.GetHeader("HX-Request") == "true"
@@ -82,7 +82,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
c.Redirect(http.StatusFound, redirectURL)
return
}
data := components.HistoryData{
History: history,
CurrentPage: page,
@@ -160,10 +160,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
// Protected routes
authorized := router.Group("/")
authorized.Use(h.AuthMiddleware())
// Password change route - only accessed from profile page
authorized.POST("/change-password", h.HandleChangePassword)
{
authorized.GET("/dashboard", h.HandleDashboard)
authorized.GET("/configs", h.HandleConfigs)
@@ -186,12 +186,16 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
authorized.GET("/profile", h.HandleProfile)
authorized.POST("/profile/theme", h.HandleUpdateTheme)
authorized.POST("/logout", h.HandleLogout)
// File metadata routes
fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
fileMetadataHandler.Register(authorized)
// AJAX routes for dashboard
authorized.GET("/dashboard/data", h.HandleDashboardData)
authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
authorized.GET("/dashboard/history", h.HandleDashboardHistoryData)
// Test connection routes
authorized.POST("/test-connection", h.HandleTestConnection)
authorized.POST("/test-sftp-connection", h.HandleTestSFTPConnection)
@@ -208,7 +212,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/users/:id", h.HandleDeleteUser)
admin.GET("/register", h.HandleRegisterPage)
admin.POST("/register", h.HandleRegister)
// Admin tools routes
admin.GET("/tools", h.HandleAdminTools)
admin.POST("/backup-database", h.HandleBackupDatabase)
@@ -222,12 +226,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
admin.DELETE("/delete-backup/:filename", h.HandleDeleteBackup)
admin.GET("/refresh-backups", h.HandleRefreshBackups)
}
// API routes
api := router.Group("/api")
{
api.POST("/login", h.HandleAPILogin)
// Protected API routes
apiAuthorized := api.Group("/")
apiAuthorized.Use(h.APIAuthMiddleware())
@@ -238,7 +242,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.POST("/configs", h.HandleAPICreateConfig)
apiAuthorized.PUT("/configs/:id", h.HandleAPIUpdateConfig)
apiAuthorized.DELETE("/configs/:id", h.HandleAPIDeleteConfig)
// Job endpoints
apiAuthorized.GET("/jobs", h.HandleAPIJobs)
apiAuthorized.GET("/jobs/:id", h.HandleAPIJob)
@@ -246,11 +250,11 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
apiAuthorized.PUT("/jobs/:id", h.HandleAPIUpdateJob)
apiAuthorized.DELETE("/jobs/:id", h.HandleAPIDeleteJob)
apiAuthorized.POST("/jobs/:id/run", h.HandleAPIRunJob)
// History endpoints
apiAuthorized.GET("/history", h.HandleAPIHistory)
apiAuthorized.GET("/job-runs/:id", h.HandleAPIJobRun)
// Admin-only API routes
apiAdmin := apiAuthorized.Group("/admin")
apiAdmin.Use(h.APIAdminMiddleware())
internal/web/middleware/auth.go
+2 -1
View File
@@ -1,3 +1,4 @@
// AuthMiddleware is a middleware function that checks if the request has a valid JWT token
func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
return func(c *gin.Context) {
@@ -41,4 +42,4 @@ func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
c.Next()
}
}
}