feat: Add file metadata tracking and search functionality

- Implement FileMetadata model to track processed files
- Create file metadata handlers for listing, searching, and viewing files
- Add file metadata routes and UI components
- Support advanced file search with multiple filters
- Enhance job execution to capture file metadata during transfers
- Implement file hash and duplicate detection logic

internal/scheduler/scheduler.go

@@ -1,12 +1,17 @@
 package scheduler
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -162,7 +167,7 @@ func (s *Scheduler) executeJob(jobID uint) {
 		"size",
 		"--include", job.Config.FilePattern,
 	}
-	
+
 	// Add source path with bucket for S3-compatible storage
 	var sourceSizePath string
 	if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -173,9 +178,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 	} else {
 		sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
 	}
-	
+
 	sizeArgs = append(sizeArgs, sourceSizePath)
-	
+
 	// Get the rclone path from the environment variable or use the default path
 	rclonePath := os.Getenv("RCLONE_PATH")
 	if rclonePath == "" {
@@ -200,7 +205,16 @@ func (s *Scheduler) executeJob(jobID uint) {
 
 	totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
 	totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
-	// totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
+	totalSize := strings.TrimSpace(strings.Split(outputStr, "Total size:")[1])
+	if strings.Contains(totalSize, "(") {
+		totalSize = strings.TrimSpace(strings.Split(totalSize, "(")[1])
+		totalSize = strings.TrimSpace(strings.Split(totalSize, " ")[0])
+	} else {
+		totalSize = "0"
+	}
+
+	bytesTransferred, _ := strconv.ParseInt(totalSize, 10, 64)
+	history.BytesTransferred = bytesTransferred
 
 	if totalObjects == "0" {
 		fmt.Printf("No files to transfer for job %d\n", jobID)
@@ -216,7 +230,7 @@ func (s *Scheduler) executeJob(jobID uint) {
 			"lsf",
 			"--include", job.Config.FilePattern,
 		}
-		
+
 		// Add source path with bucket for S3-compatible storage
 		var sourceLsPath string
 		if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -227,9 +241,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 		} else {
 			sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
 		}
-		
+
 		listArgs = append(listArgs, sourceLsPath)
-		
+
 		fmt.Printf("Listing files for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
 		// Get the rclone path from the environment variable or use the default path
 		rclonePath := os.Getenv("RCLONE_PATH")
@@ -238,29 +252,123 @@ func (s *Scheduler) executeJob(jobID uint) {
 		}
 		listCmd := exec.Command(rclonePath, listArgs...)
 		listOutput, listErr := listCmd.CombinedOutput()
-		
+
 		if listErr != nil {
 			fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
 			history.Status = "failed"
 			history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
 			return
 		}
-		
+
 		// Split the output by newlines to get individual files
 		files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
 		fmt.Printf("Found %d files to transfer for job %d\n", len(files), jobID)
-		
+
 		var transferErrors []string
 		filesTransferred := 0
-		
+
 		// Process each file individually
 		for _, file := range files {
 			if file == "" {
 				continue
 			}
-			
+
 			fmt.Printf("Processing file: %s for job %d\n", file, jobID)
-			
+
+			// Get detailed file info if this is a local source
+			var fileSize int64
+			var createTime, modTime time.Time
+			var fileHash string
+			var fileMetadataErr error
+
+			// Get file metadata based on source type
+			if job.Config.SourceType == "local" {
+				// Construct the full local path
+				localFilePath := filepath.Join(job.Config.SourcePath, file)
+
+				// Get file info (size, creation time, modification time)
+				fileSize, createTime, modTime, fileMetadataErr = getFileInfo(localFilePath)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not get file info for %s: %v\n", file, fileMetadataErr)
+				}
+
+				// Calculate file hash (MD5)
+				fileHash, fileMetadataErr = calculateFileHash(localFilePath)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not calculate hash for %s: %v\n", file, fileMetadataErr)
+				}
+
+				// Check if this file has been processed before (by hash)
+				if fileHash != "" {
+					processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+					if processed {
+						fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
+							file, fileHash, prevMetadata.FileName)
+
+						// If configured to skip previously processed files,
+						// we could add that logic here
+						// For now, we'll just log it and continue
+					}
+				}
+
+				// Also check the processing history for this specific file name
+				prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+				if histErr == nil {
+					fmt.Printf("File %s was previously processed on %s with status: %s\n",
+						file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+
+					// If the file was previously processed successfully and the hash hasn't changed,
+					// we could skip processing
+					if prevMetadata.Status == "processed" ||
+						prevMetadata.Status == "archived" ||
+						prevMetadata.Status == "deleted" ||
+						prevMetadata.Status == "archived_and_deleted" {
+						if fileHash != "" && fileHash == prevMetadata.FileHash {
+							fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", file)
+							// Skip this file and continue to the next one
+							continue
+						}
+					}
+				}
+			} else {
+				// For non-local sources, use rclone lsjson to get metadata
+				fileSize, createTime, modTime, fileHash, fileMetadataErr = s.getRemoteFileInfo(&job.Config, file)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not get remote file info for %s: %v\n", file, fileMetadataErr)
+					// We'll continue with placeholder values
+					fileSize = 0
+					createTime = time.Now()
+					modTime = time.Now()
+					fileHash = ""
+				} else {
+					// If we got a hash, check for previous processing
+					if fileHash != "" {
+						processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+						if processed {
+							fmt.Printf("Remote file %s has been processed before (hash: %s, previous file: %s)\n",
+								file, fileHash, prevMetadata.FileName)
+
+							// Skip previously processed files with the same hash if they were processed successfully
+							if prevMetadata.Status == "processed" ||
+								prevMetadata.Status == "archived" ||
+								prevMetadata.Status == "deleted" ||
+								prevMetadata.Status == "archived_and_deleted" {
+								fmt.Printf("Skipping unchanged remote file %s (hash matches previous processing)\n", file)
+								// Skip this file and continue to the next one
+								continue
+							}
+						}
+					}
+
+					// Also check by filename
+					prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+					if histErr == nil {
+						fmt.Printf("Remote file %s was previously processed on %s with status: %s\n",
+							file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+					}
+				}
+			}
+
 			// Prepare moveto command for transfer
 			transferArgs := []string{
 				"--config", configPath,
@@ -270,10 +378,10 @@ func (s *Scheduler) executeJob(jobID uint) {
 				"--verbose",
 				"--stats", "1s",
 			}
-			
+
 			// Source and destination paths
 			var sourcePath, destPath string
-			
+
 			// For S3, MinIO, and B2, include the bucket in the path
 			if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
 				sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, file)
@@ -283,7 +391,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 			} else {
 				sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
 			}
-			
+
+			var destFile string = file
+
 			if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
 				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
 				if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
@@ -292,27 +402,36 @@ func (s *Scheduler) executeJob(jobID uint) {
 			} else {
 				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, file)
 			}
-			
+
 			// Add output filename pattern if specified
 			if job.Config.OutputPattern != "" {
 				// Process the output pattern for this specific file
-				newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
-				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
-				fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
+				destFile = ProcessOutputPattern(job.Config.OutputPattern, file)
+
+				if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+					destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
+					if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+						destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
+					}
+				} else {
+					destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
+				}
+
+				fmt.Printf("Renaming file from %s to %s for job %d\n", file, destFile, jobID)
 			}
-			
+
 			// Add custom flags if specified
 			if job.Config.RcloneFlags != "" {
 				customFlags := strings.Split(job.Config.RcloneFlags, " ")
 				transferArgs = append(transferArgs, customFlags...)
 				fmt.Printf("Added custom flags for job %d: %v\n", jobID, customFlags)
 			}
-			
+
 			// Add source and destination to the command
 			transferArgs = append(transferArgs, sourcePath, destPath)
-			
+
 			// Execute transfer for this file
-			fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n", 
+			fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
 				jobID, file, strings.Join(transferArgs, " "))
 			// Get the rclone path from the environment variable or use the default path
 			rclonePath := os.Getenv("RCLONE_PATH")
@@ -325,25 +444,48 @@ func (s *Scheduler) executeJob(jobID uint) {
 			// Print the output
 			fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
 
+			// Create file metadata record
+			fileStatus := "processed"
+			var fileErrorMsg string
+			var destPathForDB string
+
 			// Check if file was successfully transferred
 			if fileErr != nil {
 				fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
 				transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
+				fileStatus = "error"
+				fileErrorMsg = fileErr.Error()
 			} else {
 				filesTransferred++
 				fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
-				
+
+				// Extract the actual destination path (without rclone remote prefix)
+				if job.Config.DestinationType == "local" {
+					destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
+				} else {
+					// For remote destinations, store the path format
+					if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+						if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+							destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
+						} else {
+							destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
+						}
+					} else {
+						destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
+					}
+				}
+
 				// If archiving is enabled and transfer was successful, move files to archive
 				if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
 					fmt.Printf("Archiving file %s for job %d\n", file, jobID)
-					
+
 					// We don't need to move the file since we used moveto, but we can copy it to archive
 					archiveArgs := []string{
 						"--config", configPath,
 						"copyto",
 						sourcePath,
 					}
-					
+
 					// Construct archive path with bucket if needed
 					var archiveDest string
 					if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -351,10 +493,10 @@ func (s *Scheduler) executeJob(jobID uint) {
 					} else {
 						archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, file)
 					}
-					
+
 					archiveArgs = append(archiveArgs, archiveDest)
-					
-					fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n", 
+
+					fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
 						jobID, file, strings.Join(archiveArgs, " "))
 					// Get the rclone path from the environment variable or use the default path
 					rclonePath := os.Getenv("RCLONE_PATH")
@@ -370,34 +512,64 @@ func (s *Scheduler) executeJob(jobID uint) {
 					// Check if file was successfully transferred
 					if archiveErr != nil {
 						fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
-						transferErrors = append(transferErrors, 
+						transferErrors = append(transferErrors,
 							fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
+					} else {
+						fileStatus = "archived"
 					}
 				}
+
 				if job.Config.DeleteAfterTransfer {
 					fmt.Printf("Deleting file %s for job %d\n", file, jobID)
 					deleteArgs := []string{
 						"--config", configPath,
 						"deletefile",
-						sourcePath,					}
+						sourcePath}
 					deleteCmd := exec.Command(rclonePath, deleteArgs...)
 					deleteOutput, deleteErr := deleteCmd.CombinedOutput()
 					fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
 					if deleteErr != nil {
 						fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
-						transferErrors = append(transferErrors, 
+						transferErrors = append(transferErrors,
 							fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
+					} else {
+						if fileStatus == "archived" {
+							fileStatus = "archived_and_deleted"
+						} else {
+							fileStatus = "deleted"
+						}
 					}
 				}
 			}
+
+			// Create and save file metadata
+			metadata := &db.FileMetadata{
+				JobID:           jobID,
+				FileName:        file,
+				OriginalPath:    job.Config.SourcePath,
+				FileSize:        fileSize,
+				FileHash:        fileHash,
+				CreationTime:    createTime,
+				ModTime:         modTime,
+				ProcessedTime:   time.Now(),
+				DestinationPath: destPathForDB,
+				Status:          fileStatus,
+				ErrorMessage:    fileErrorMsg,
+			}
+
+			if err := s.db.CreateFileMetadata(metadata); err != nil {
+				fmt.Printf("Error creating file metadata for %s: %v\n", file, err)
+			} else {
+				fmt.Printf("Created file metadata record for %s (ID: %d)\n", file, metadata.ID)
+			}
 		}
-		
+
 		// Update job history with transfer results
 		history.FilesTransferred = filesTransferred
-		
+
 		if len(transferErrors) > 0 {
 			history.Status = "completed_with_errors"
-			history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s", 
+			history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
 				len(transferErrors), strings.Join(transferErrors, "\n"))
 		}
 	}
@@ -439,15 +611,15 @@ func ProcessOutputPattern(pattern string, originalFilename string) string {
 		format := dateRegex.FindStringSubmatch(match)[1]
 		return time.Now().Format(format)
 	})
-	
+
 	// Split the filename and extension
 	ext := filepath.Ext(originalFilename)
 	filename := strings.TrimSuffix(originalFilename, ext)
-	
+
 	// Replace filename and extension variables
 	processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
 	processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
-	
+
 	return processedPattern
 }
 
@@ -467,31 +639,31 @@ func createRcloneFilterFile(pattern string) (string, error) {
 		format := dateRegex.FindStringSubmatch(match)[1]
 		return time.Now().Format(format)
 	})
-	
+
 	// Replace filename and extension variables with rclone's capture group references
 	// For rclone rename filters, we need to use {1} for the first capture group, not $1
 	// See: https://rclone.org/filtering/#rename
-	
+
 	// Extract filename without extension
 	processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
-	
+
 	// Extract extension (with the dot)
 	processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
-	
+
 	// Create a rename rule for rclone using the correct syntax:
 	// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
 	// - For files with extension: capture the name and extension separately
 	rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
-	
+
 	// Add a fallback rule for files without extension
-	fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n", 
+	fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
 		strings.ReplaceAll(processedPattern, "{2}", ""))
-	
+
 	// Write the rules to the file
 	if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
 		return "", fmt.Errorf("failed to write to filter file: %v", err)
 	}
-	
+
 	return tmpFile.Name(), nil
 }
 
@@ -515,3 +687,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
 	go s.executeJob(jobID)
 	return nil
 }
+
+// calculateFileHash computes an MD5 hash for the given file path
+func calculateFileHash(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", fmt.Errorf("error opening file: %v", err)
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return "", fmt.Errorf("error calculating hash: %v", err)
+	}
+
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// getFileInfo retrieves file stats like size, creation time, and modification time
+func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
+	info, err := os.Stat(filePath)
+	if err != nil {
+		return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
+	}
+
+	size := info.Size()
+	modTime := info.ModTime()
+
+	// Get creation time (this is platform-specific)
+	// For simplicity, we'll use modification time as a fallback
+	createTime := modTime
+
+	return size, createTime, modTime, nil
+}
+
+// hasFileBeenProcessed checks if a file with the same hash has been processed before
+func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
+	if fileHash == "" {
+		return false, nil, nil
+	}
+
+	// First try to find by hash (most reliable)
+	metadata, err := s.db.GetFileMetadataByHash(fileHash)
+	if err == nil && metadata != nil {
+		return true, metadata, nil
+	}
+
+	return false, nil, nil
+}
+
+// checkFileProcessingHistory checks processing history for a given file
+func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
+	// Try to find by job and filename
+	metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
+	if err == nil && metadata != nil {
+		return metadata, nil
+	}
+
+	return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
+}
+
+// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
+func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
+	// Get rclone config path
+	configPath := s.db.GetConfigRclonePath(config)
+
+	// Construct the appropriate source path
+	var sourcePath string
+	if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
+		if config.SourcePath != "" && config.SourcePath != "/" {
+			sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
+		}
+	} else {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
+	}
+
+	// Use rclone lsjson to get file details
+	rclonePath := os.Getenv("RCLONE_PATH")
+	if rclonePath == "" {
+		rclonePath = "rclone"
+	}
+
+	// Construct the full path to the file
+	fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
+
+	// Run rclone lsjson command
+	args := []string{
+		"--config", configPath,
+		"lsjson",
+		"--hash",
+		fullPath,
+	}
+
+	cmd := exec.Command(rclonePath, args...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
+	}
+
+	// Parse the JSON output
+	var files []map[string]interface{}
+	if err := json.Unmarshal(output, &files); err != nil {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
+	}
+
+	if len(files) == 0 {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
+	}
+
+	fileInfo := files[0]
+
+	// Extract file size
+	var fileSize int64
+	if size, ok := fileInfo["Size"].(float64); ok {
+		fileSize = int64(size)
+	}
+
+	// Extract modification time
+	modTime := time.Now()
+	if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
+		if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
+			modTime = parsedTime
+		}
+	}
+
+	// Create time is usually not available for remote files, so we'll use modTime
+	createTime := modTime
+
+	// Calculate hash if available
+	var md5Hash string
+	if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
+		if md5, ok := hashes["md5"].(string); ok {
+			md5Hash = md5
+		}
+	}
+
+	return fileSize, createTime, modTime, md5Hash, nil
+}