feat: Add file metadata tracking and search functionality

- Implement FileMetadata model to track processed files
- Create file metadata handlers for listing, searching, and viewing files
- Add file metadata routes and UI components
- Support advanced file search with multiple filters
- Enhance job execution to capture file metadata during transfers
- Implement file hash and duplicate detection logic

components/file_metadata.templ

@@ -0,0 +1,587 @@
+package components
+
+import (
+	"context"
+	"strconv"
+	"fmt"
+	"github.com/starfleetcptn/gomft/internal/db"
+)
+
+// FileMetadataFilter represents filter parameters for file metadata queries
+type FileMetadataFilter struct {
+	Status    string
+	JobID     string
+	FileName  string
+	Hash      string
+	StartDate string
+	EndDate   string
+}
+
+// FileMetadataListData contains data for the file metadata list template
+type FileMetadataListData struct {
+	Files      []db.FileMetadata
+	TotalCount int64
+	Page       int
+	Limit      int
+	TotalPages int
+	Job        *db.Job // Optional: if viewing files for a specific job
+	Filter     FileMetadataFilter
+}
+
+// FileMetadataDetailsData contains data for the file metadata details template
+type FileMetadataDetailsData struct {
+	File db.FileMetadata
+}
+
+// FileMetadataSearchData contains data for the file metadata search template
+type FileMetadataSearchData struct {
+	Files      []db.FileMetadata
+	TotalCount int64
+	Page       int
+	Limit      int
+	TotalPages int
+	Filter     FileMetadataFilter
+}
+
+// getStatusBadgeClass returns the appropriate CSS class for a file status badge
+func getStatusBadgeClass(status string) string {
+	switch status {
+	case "processed":
+		return "badge-success"
+	case "archived":
+		return "badge-info"
+	case "deleted":
+		return "badge-warning"
+	case "archived_and_deleted":
+		return "badge-warning"
+	case "error":
+		return "badge-danger"
+	default:
+		return "badge-secondary"
+	}
+}
+
+// formatFileSize formats a file size in bytes to a human-readable string
+func formatFileSize(size int64) string {
+	if size < 1024 {
+		return fmt.Sprintf("%d B", size)
+	} else if size < 1024*1024 {
+		return fmt.Sprintf("%.2f KB", float64(size)/1024)
+	} else if size < 1024*1024*1024 {
+		return fmt.Sprintf("%.2f MB", float64(size)/(1024*1024))
+	} else {
+		return fmt.Sprintf("%.2f GB", float64(size)/(1024*1024*1024))
+	}
+}
+
+// FileMetadataList renders a list of file metadata with pagination and filters
+templ FileMetadataList(ctx context.Context, data FileMetadataListData) {
+	@LayoutWithContext("File Metadata", ctx) {
+		<div class="container mx-auto px-4 py-8 animate-fadeIn">
+			<div class="flex justify-between items-center mb-6">
+				<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
+					if data.Job != nil {
+						Files for Job: { data.Job.Name }
+					} else {
+						File Metadata
+					}
+				</h1>
+				<div class="flex space-x-2">
+					<a href="/files/search" class="btn-primary">
+						<i class="fas fa-search mr-2"></i> Advanced Search
+					</a>
+				</div>
+			</div>
+			
+			<!-- Filter Form -->
+			<div class="card mb-6">
+				<div class="card-header">
+					<h2 class="text-lg font-semibold">Filter Files</h2>
+				</div>
+				<div class="card-body">
+					<form method="GET" class="grid grid-cols-1 md:grid-cols-3 gap-4">
+						if data.Job == nil {
+							<div>
+								<label for="job_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job</label>
+								<input type="text" id="job_id" name="job_id" value={ data.Filter.JobID } placeholder="Job ID" 
+									class="form-input w-full" />
+							</div>
+						}
+						<div>
+							<label for="status" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Status</label>
+							<select id="status" name="status" class="form-input w-full">
+								<option value="">All Statuses</option>
+								<option value="processed" selected?={ data.Filter.Status == "processed" }>Processed</option>
+								<option value="archived" selected?={ data.Filter.Status == "archived" }>Archived</option>
+								<option value="deleted" selected?={ data.Filter.Status == "deleted" }>Deleted</option>
+								<option value="archived_and_deleted" selected?={ data.Filter.Status == "archived_and_deleted" }>Archived & Deleted</option>
+								<option value="error" selected?={ data.Filter.Status == "error" }>Error</option>
+							</select>
+						</div>
+						<div>
+							<label for="filename" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Filename</label>
+							<input type="text" id="filename" name="filename" value={ data.Filter.FileName } placeholder="Filename or partial match" 
+								class="form-input w-full" />
+						</div>
+						<div class="md:col-span-3 flex justify-end">
+							<button type="submit" class="btn-primary">
+								<i class="fas fa-filter mr-2"></i> Apply Filters
+							</button>
+						</div>
+					</form>
+				</div>
+			</div>
+			
+			<!-- Results -->
+			<div class="card">
+				<div class="card-header flex justify-between items-center">
+					<h2 class="text-lg font-semibold">Files ({ strconv.FormatInt(data.TotalCount, 10) })</h2>
+					<div class="text-sm text-secondary-600 dark:text-secondary-400">
+						Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) }
+					</div>
+				</div>
+				<div class="overflow-x-auto">
+					if len(data.Files) > 0 {
+						<table class="table w-full">
+							<thead>
+								<tr>
+									<th>ID</th>
+									<th>Filename</th>
+									<th>Size</th>
+									<th>Processed</th>
+									<th>Status</th>
+									if data.Job == nil {
+										<th>Job</th>
+									}
+									<th>Actions</th>
+								</tr>
+							</thead>
+							<tbody>
+								for _, file := range data.Files {
+									<tr class="hover:bg-secondary-50 dark:hover:bg-secondary-800 transition-colors duration-150">
+										<td>{ strconv.FormatUint(uint64(file.ID), 10) }</td>
+										<td class="text-primary-600 dark:text-primary-400">
+											<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="hover:underline">
+												{ file.FileName }
+											</a>
+										</td>
+										<td>{ formatFileSize(file.FileSize) }</td>
+										<td>{ file.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
+										<td>
+											<span class={ "badge", getStatusBadgeClass(file.Status) }>
+												{ file.Status }
+											</span>
+										</td>
+										if data.Job == nil && file.Job.ID > 0 {
+											<td>
+												<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", file.Job.ID)) } class="hover:underline text-primary-600 dark:text-primary-400">
+													{ file.Job.Name }
+												</a>
+											</td>
+										}
+										<td class="flex space-x-2">
+											<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="text-primary-600 dark:text-primary-400 hover:text-primary-800 dark:hover:text-primary-300">
+												<i class="fas fa-eye"></i>
+											</a>
+											<button
+												class="text-red-600 dark:text-red-400 hover:text-red-800 dark:hover:text-red-300"
+												hx-delete={ fmt.Sprintf("/files/%d", file.ID) }
+												hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
+												hx-target="body">
+												<i class="fas fa-trash"></i>
+											</button>
+										</td>
+									</tr>
+								}
+							</tbody>
+						</table>
+						
+						<!-- Pagination -->
+						if data.TotalPages > 1 {
+							<div class="flex justify-center items-center py-4 bg-secondary-50 dark:bg-secondary-800">
+								<div class="flex space-x-1">
+									if data.Page > 1 {
+										<a href={ templ.SafeURL(buildPaginationURL(data, data.Page - 1)) } class="btn-secondary">
+											<i class="fas fa-chevron-left"></i>
+										</a>
+									}
+									
+									for i := 1; i <= data.TotalPages; i++ {
+										if i == data.Page {
+											<span class="px-3 py-2 bg-primary-600 text-white rounded">{ strconv.Itoa(i) }</span>
+										} else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) {
+											<a href={ templ.SafeURL(buildPaginationURL(data, i)) } class="px-3 py-2 bg-secondary-200 text-secondary-800 hover:bg-secondary-300 rounded dark:bg-secondary-700 dark:text-secondary-200 dark:hover:bg-secondary-600">
+												{ strconv.Itoa(i) }
+											</a>
+										} else if i == data.Page-3 || i == data.Page+3 {
+											<span class="px-3 py-2">...</span>
+										}
+									}
+									
+									if data.Page < data.TotalPages {
+										<a href={ templ.SafeURL(buildPaginationURL(data, data.Page + 1)) } class="btn-secondary">
+											<i class="fas fa-chevron-right"></i>
+										</a>
+									}
+								</div>
+							</div>
+						}
+					} else {
+						<div class="p-6 text-center text-secondary-600 dark:text-secondary-400">
+							<i class="fas fa-file-alt text-5xl mb-3"></i>
+							<p>No file metadata found matching your criteria.</p>
+						</div>
+					}
+				</div>
+			</div>
+		</div>
+	}
+}
+
+// buildPaginationURL builds a URL for pagination links
+func buildPaginationURL(data FileMetadataListData, page int) string {
+	baseURL := "/files"
+	if data.Job != nil {
+		baseURL = fmt.Sprintf("/files/job/%d", data.Job.ID)
+	}
+	
+	url := fmt.Sprintf("%s?page=%d&limit=%d", baseURL, page, data.Limit)
+	
+	if data.Filter.Status != "" {
+		url += "&status=" + data.Filter.Status
+	}
+	
+	if data.Filter.FileName != "" {
+		url += "&filename=" + data.Filter.FileName
+	}
+	
+	if data.Filter.JobID != "" && data.Job == nil {
+		url += "&job_id=" + data.Filter.JobID
+	}
+	
+	return url
+}
+
+// FileMetadataDetails renders detailed information about a file
+templ FileMetadataDetails(ctx context.Context, data FileMetadataDetailsData) {
+	@LayoutWithContext("File Details", ctx) {
+		<div class="container mx-auto px-4 py-8 animate-fadeIn">
+			<div class="mb-6">
+				<a href="/files" class="text-primary-600 dark:text-primary-400 hover:underline">
+					<i class="fas fa-arrow-left mr-2"></i> Back to Files
+				</a>
+			</div>
+			
+			<div class="card">
+				<div class="card-header">
+					<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
+						File: { data.File.FileName }
+					</h1>
+				</div>
+				<div class="card-body">
+					<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
+						<div>
+							<h2 class="text-lg font-semibold mb-4 text-secondary-800 dark:text-secondary-200">File Information</h2>
+							<table class="w-full">
+								<tbody>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">ID</td>
+										<td class="py-2">{ strconv.FormatUint(uint64(data.File.ID), 10) }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Filename</td>
+										<td class="py-2">{ data.File.FileName }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Size</td>
+										<td class="py-2">{ formatFileSize(data.File.FileSize) }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Hash</td>
+										<td class="py-2 break-all">
+											if data.File.FileHash != "" {
+												{ data.File.FileHash }
+											} else {
+												<span class="text-secondary-500 dark:text-secondary-500 italic">Not available</span>
+											}
+										</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Status</td>
+										<td class="py-2">
+											<span class={ "badge", getStatusBadgeClass(data.File.Status) }>
+												{ data.File.Status }
+											</span>
+										</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Original Path</td>
+										<td class="py-2 break-all">{ data.File.OriginalPath }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Destination Path</td>
+										<td class="py-2 break-all">{ data.File.DestinationPath }</td>
+									</tr>
+								</tbody>
+							</table>
+						</div>
+						
+						<div>
+							<h2 class="text-lg font-semibold mb-4 text-secondary-800 dark:text-secondary-200">Processing Information</h2>
+							<table class="w-full">
+								<tbody>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Job</td>
+										<td class="py-2">
+											<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", data.File.JobID)) } class="text-primary-600 dark:text-primary-400 hover:underline">
+												{ data.File.Job.Name }
+											</a>
+										</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Processed Time</td>
+										<td class="py-2">{ data.File.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Creation Time</td>
+										<td class="py-2">{ data.File.CreationTime.Format("2006-01-02 15:04:05") }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Modification Time</td>
+										<td class="py-2">{ data.File.ModTime.Format("2006-01-02 15:04:05") }</td>
+									</tr>
+									if data.File.Status == "error" && data.File.ErrorMessage != "" {
+										<tr class="border-b border-secondary-200 dark:border-secondary-700">
+											<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Error</td>
+											<td class="py-2 text-red-600 dark:text-red-400 break-all">{ data.File.ErrorMessage }</td>
+										</tr>
+									}
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Record Created</td>
+										<td class="py-2">{ data.File.CreatedAt.Format("2006-01-02 15:04:05") }</td>
+									</tr>
+									<tr class="border-b border-secondary-200 dark:border-secondary-700">
+										<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Record Updated</td>
+										<td class="py-2">{ data.File.UpdatedAt.Format("2006-01-02 15:04:05") }</td>
+									</tr>
+								</tbody>
+							</table>
+						</div>
+					</div>
+					
+					<div class="mt-8 flex justify-end space-x-4">
+						<a href="/files" class="btn-secondary">
+							<i class="fas fa-list mr-2"></i> Back to Files
+						</a>
+						<button 
+							class="btn-danger"
+							hx-delete={ fmt.Sprintf("/files/%d", data.File.ID) }
+							hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
+							hx-target="body">
+							<i class="fas fa-trash mr-2"></i> Delete Record
+						</button>
+					</div>
+				</div>
+			</div>
+		</div>
+	}
+}
+
+// FileMetadataSearch renders an advanced search form for file metadata
+templ FileMetadataSearch(ctx context.Context, data FileMetadataSearchData) {
+	@LayoutWithContext("Search Files", ctx) {
+		<div class="container mx-auto px-4 py-8 animate-fadeIn">
+			<div class="mb-6">
+				<a href="/files" class="text-primary-600 dark:text-primary-400 hover:underline">
+					<i class="fas fa-arrow-left mr-2"></i> Back to Files
+				</a>
+			</div>
+			
+			<div class="card mb-6">
+				<div class="card-header">
+					<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
+						Advanced File Search
+					</h1>
+				</div>
+				<div class="card-body">
+					<form method="GET" class="grid grid-cols-1 md:grid-cols-2 gap-6">
+						<div>
+							<label for="job_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job ID</label>
+							<input type="text" id="job_id" name="job_id" value={ data.Filter.JobID } placeholder="Job ID" 
+								class="form-input w-full" />
+						</div>
+						<div>
+							<label for="status" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Status</label>
+							<select id="status" name="status" class="form-input w-full">
+								<option value="">All Statuses</option>
+								<option value="processed" selected?={ data.Filter.Status == "processed" }>Processed</option>
+								<option value="archived" selected?={ data.Filter.Status == "archived" }>Archived</option>
+								<option value="deleted" selected?={ data.Filter.Status == "deleted" }>Deleted</option>
+								<option value="archived_and_deleted" selected?={ data.Filter.Status == "archived_and_deleted" }>Archived & Deleted</option>
+								<option value="error" selected?={ data.Filter.Status == "error" }>Error</option>
+							</select>
+						</div>
+						<div>
+							<label for="filename" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Filename</label>
+							<input type="text" id="filename" name="filename" value={ data.Filter.FileName } placeholder="Filename or partial match" 
+								class="form-input w-full" />
+						</div>
+						<div>
+							<label for="hash" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">File Hash</label>
+							<input type="text" id="hash" name="hash" value={ data.Filter.Hash } placeholder="MD5 hash" 
+								class="form-input w-full" />
+						</div>
+						<div>
+							<label for="start_date" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Processed After</label>
+							<input type="date" id="start_date" name="start_date" value={ data.Filter.StartDate } 
+								class="form-input w-full" />
+						</div>
+						<div>
+							<label for="end_date" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Processed Before</label>
+							<input type="date" id="end_date" name="end_date" value={ data.Filter.EndDate } 
+								class="form-input w-full" />
+						</div>
+						<div class="md:col-span-2 flex justify-end">
+							<button type="submit" class="btn-primary">
+								<i class="fas fa-search mr-2"></i> Search Files
+							</button>
+						</div>
+					</form>
+				</div>
+			</div>
+			
+			<!-- Results -->
+			if data.TotalCount > 0 || (data.Filter.FileName != "" || data.Filter.JobID != "" || data.Filter.Status != "" || 
+				data.Filter.Hash != "" || data.Filter.StartDate != "" || data.Filter.EndDate != "") {
+				<div class="card">
+					<div class="card-header flex justify-between items-center">
+						<h2 class="text-lg font-semibold">Search Results ({ strconv.FormatInt(data.TotalCount, 10) })</h2>
+						<div class="text-sm text-secondary-600 dark:text-secondary-400">
+							Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) }
+						</div>
+					</div>
+					<div class="overflow-x-auto">
+						if len(data.Files) > 0 {
+							<table class="table w-full">
+								<thead>
+									<tr>
+										<th>ID</th>
+										<th>Filename</th>
+										<th>Size</th>
+										<th>Processed</th>
+										<th>Status</th>
+										<th>Job</th>
+										<th>Actions</th>
+									</tr>
+								</thead>
+								<tbody>
+									for _, file := range data.Files {
+										<tr class="hover:bg-secondary-50 dark:hover:bg-secondary-800 transition-colors duration-150">
+											<td>{ strconv.FormatUint(uint64(file.ID), 10) }</td>
+											<td class="text-primary-600 dark:text-primary-400">
+												<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="hover:underline">
+													{ file.FileName }
+												</a>
+											</td>
+											<td>{ formatFileSize(file.FileSize) }</td>
+											<td>{ file.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
+											<td>
+												<span class={ "badge", getStatusBadgeClass(file.Status) }>
+													{ file.Status }
+												</span>
+											</td>
+											<td>
+												<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", file.Job.ID)) } class="hover:underline text-primary-600 dark:text-primary-400">
+													{ file.Job.Name }
+												</a>
+											</td>
+											<td class="flex space-x-2">
+												<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="text-primary-600 dark:text-primary-400 hover:text-primary-800 dark:hover:text-primary-300">
+													<i class="fas fa-eye"></i>
+												</a>
+												<button
+													class="text-red-600 dark:text-red-400 hover:text-red-800 dark:hover:text-red-300"
+													hx-delete={ fmt.Sprintf("/files/%d", file.ID) }
+													hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
+													hx-target="body">
+													<i class="fas fa-trash"></i>
+												</button>
+											</td>
+										</tr>
+									}
+								</tbody>
+							</table>
+							
+							<!-- Pagination -->
+							if data.TotalPages > 1 {
+								<div class="flex justify-center items-center py-4 bg-secondary-50 dark:bg-secondary-800">
+									<div class="flex space-x-1">
+										if data.Page > 1 {
+											<a href={ templ.SafeURL(buildSearchPaginationURL(data, data.Page - 1)) } class="btn-secondary">
+												<i class="fas fa-chevron-left"></i>
+											</a>
+										}
+										
+										for i := 1; i <= data.TotalPages; i++ {
+											if i == data.Page {
+												<span class="px-3 py-2 bg-primary-600 text-white rounded">{ strconv.Itoa(i) }</span>
+											} else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) {
+												<a href={ templ.SafeURL(buildSearchPaginationURL(data, i)) } class="px-3 py-2 bg-secondary-200 text-secondary-800 hover:bg-secondary-300 rounded dark:bg-secondary-700 dark:text-secondary-200 dark:hover:bg-secondary-600">
+													{ strconv.Itoa(i) }
+												</a>
+											} else if i == data.Page-3 || i == data.Page+3 {
+												<span class="px-3 py-2">...</span>
+											}
+										}
+										
+										if data.Page < data.TotalPages {
+											<a href={ templ.SafeURL(buildSearchPaginationURL(data, data.Page + 1)) } class="btn-secondary">
+												<i class="fas fa-chevron-right"></i>
+											</a>
+										}
+									</div>
+								</div>
+							}
+						} else {
+							<div class="p-6 text-center text-secondary-600 dark:text-secondary-400">
+								<i class="fas fa-search text-5xl mb-3"></i>
+								<p>No files found matching your search criteria.</p>
+							</div>
+						}
+					</div>
+				</div>
+			}
+		</div>
+	}
+}
+
+// buildSearchPaginationURL builds a URL for search pagination links
+func buildSearchPaginationURL(data FileMetadataSearchData, page int) string {
+	url := fmt.Sprintf("/files/search?page=%d&limit=%d", page, data.Limit)
+	
+	if data.Filter.Status != "" {
+		url += "&status=" + data.Filter.Status
+	}
+	
+	if data.Filter.FileName != "" {
+		url += "&filename=" + data.Filter.FileName
+	}
+	
+	if data.Filter.JobID != "" {
+		url += "&job_id=" + data.Filter.JobID
+	}
+	
+	if data.Filter.Hash != "" {
+		url += "&hash=" + data.Filter.Hash
+	}
+	
+	if data.Filter.StartDate != "" {
+		url += "&start_date=" + data.Filter.StartDate
+	}
+	
+	if data.Filter.EndDate != "" {
+		url += "&end_date=" + data.Filter.EndDate
+	}
+	
+	return url
+} 

components/layout.templ

@@ -300,6 +300,9 @@ templ LayoutWithContext(title string, ctx context.Context) {
 									<a href="/history" class="nav-link">
 										<i class="fas fa-history mr-1"></i> History
 									</a>
+									<a href="/files" class="nav-link">
+										<i class="fas fa-file-alt mr-1"></i> Files
+									</a>
 									if isAdmin(ctx) {
 										<a href="/admin/users" class="nav-link">
 											<i class="fas fa-users mr-1"></i> Users
@@ -393,6 +396,9 @@ templ LayoutWithContext(title string, ctx context.Context) {
 						<a href="/history" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
 							<i class="fas fa-history mr-2"></i> History
 						</a>
+						<a href="/files" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
+							<i class="fas fa-file-alt mr-2"></i> Files
+						</a>
 						if isAdmin(ctx) {
 							<a href="/admin/users" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
 								<i class="fas fa-users mr-2"></i> Users
@@ -446,6 +452,10 @@ templ LayoutWithContext(title string, ctx context.Context) {
 							<i class="fas fa-history text-lg"></i>
 							<span class="text-xs mt-1">History</span>
 						</a>
+						<a href="/files" class="flex flex-col items-center justify-center text-secondary-500 dark:text-secondary-400 hover:text-primary-600 dark:hover:text-primary-400">
+							<i class="fas fa-file-alt text-lg"></i>
+							<span class="text-xs mt-1">Files</span>
+						</a>
 						<button 
 							type="button" 
 							class="flex flex-col items-center justify-center text-secondary-500 dark:text-secondary-400 hover:text-primary-600 dark:hover:text-primary-400"

components/providers/common/common.templ

@@ -65,45 +65,63 @@ templ FilePatternFields() {
 }
 
 templ ArchiveOptions() {
-<div class="space-y-6">
+	<div class="border border-gray-200 rounded-lg p-6 bg-gray-50 shadow-sm">
-	<div class="flex items-start mb-4">
+		<h3 class="text-lg font-medium text-gray-900 mb-4">Archive &amp; Delete Options</h3>
-		<div class="flex items-center h-5">
+		
-			<input id="archive_enabled" name="archive_enabled" type="checkbox" x-model="archiveEnabled"
+		<div class="mb-4">
-				class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
+			<label for="archive_enabled" class="flex items-center cursor-pointer">
+				<div class="relative">
+					<input id="archive_enabled" name="archive_enabled" type="checkbox" x-model="archiveEnabled"
+						class="sr-only"
+						:value="archiveEnabled ? 'true' : 'false'"
+					/>
+					<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
+					<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
+						:class="archiveEnabled ? 'transform translate-x-6 bg-primary-500' : ''"></div>
+				</div>
+				<div class="ml-3 text-gray-700 font-medium">
+					Enable archiving
+				</div>
+			</label>
 		</div>
-		<div class="ml-3 text-sm">
+		
-			<label for="archive_enabled" class="font-medium text-secondary-700 dark:text-secondary-300">Enable
+		<div class="mb-4" x-show="archiveEnabled">
-				Archiving</label>
+			<label for="archive_path" class="block text-sm font-medium text-gray-700 mb-1">Archive Path</label>
-			<p class="text-secondary-500 dark:text-secondary-400">Archive files after successful transfer</p>
+			<div class="relative">
-		</div>
+				<span class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
-	</div>
+					<i class="fas fa-folder text-gray-400"></i>
-
+				</span>
-	<div x-show="archiveEnabled" class="sm:col-span-4">
+				<input id="archive_path" name="archive_path" type="text" 
-		<label for="archive_path"
+					class="pl-10 pr-10 py-2 border border-gray-300 rounded-md focus:ring-primary-500 focus:border-primary-500 block w-full shadow-sm"
-			class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Archive Path</label>
+					placeholder="Path to archive files"
-		<div class="relative">
+					x-model="archivePath"
-			<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
+				/>
-				<i class="fas fa-archive text-secondary-400 dark:text-secondary-600"></i>
 			</div>
-			<input type="text" name="archive_path" id="archive_path" x-model="archivePath"
+			<p class="mt-1 text-xs text-gray-500">
-				x-bind:required="archiveEnabled"
+				Files will be moved here after successful transfer
-				class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
+			</p>
-				placeholder="/path/to/archive" />
+		</div>
+		
+		<div class="mb-4">
+			<label for="delete_after_transfer" class="flex items-center cursor-pointer">
+				<div class="relative">
+					<input id="delete_after_transfer" name="delete_after_transfer" type="checkbox" x-model="deleteAfterTransfer"
+						class="sr-only"
+						:value="deleteAfterTransfer ? 'true' : 'false'"
+					/>
+					<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
+					<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
+						:class="deleteAfterTransfer ? 'transform translate-x-6 bg-red-500' : ''"></div>
+				</div>
+				<div class="ml-3 text-gray-700 font-medium">
+					Delete source files after transfer
+				</div>
+			</label>
+			<p class="mt-1 ml-14 text-xs text-red-500" x-show="deleteAfterTransfer">
+				<i class="fas fa-exclamation-triangle mr-1"></i> Warning: This will permanently delete the original files
+			</p>
 		</div>
 	</div>
-
-	<div class="flex items-start mb-4">
-		<div class="flex items-center h-5">
-			<input id="delete_after_transfer" name="delete_after_transfer" type="checkbox" x-model="deleteAfterTransfer"
-				class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
-		</div>
-		<div class="ml-3 text-sm">
-			<label for="delete_after_transfer" class="font-medium text-secondary-700 dark:text-secondary-300">Delete
-				After Transfer</label>
-			<p class="text-secondary-500 dark:text-secondary-400">Delete source files after successful transfer</p>
-		</div>
-	</div>
-</div>
 }
 
 templ RcloneFlags() {

components/providers/destination/ftp.templ

@@ -94,6 +94,7 @@ templ FTPDestinationForm() {
 					name="dest_passive_mode" 
 					type="checkbox" 
 					x-model="destPassiveMode"
+					:value="destPassiveMode ? 'true' : 'false'"
 					class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
 			</div>
 			<div class="ml-3 text-sm">

components/providers/source/ftp.templ

@@ -94,6 +94,7 @@ templ FTPSourceForm() {
 					name="source_passive_mode" 
 					type="checkbox" 
 					x-model="sourcePassiveMode"
+					:value="sourcePassiveMode ? 'true' : 'false'"
 					class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
 			</div>
 			<div class="ml-3 text-sm">

internal/db/db.go

@@ -7,23 +7,23 @@ import (
 	"path/filepath"
 	"time"
 
-	"github.com/starfleetcptn/gomft/internal/auth"
 	"github.com/glebarez/sqlite"
+	"github.com/starfleetcptn/gomft/internal/auth"
 	"gorm.io/gorm"
 )
 
 type User struct {
-	ID                 uint   `gorm:"primarykey"`
+	ID                  uint   `gorm:"primarykey"`
-	Email              string `gorm:"unique;not null"`
+	Email               string `gorm:"unique;not null"`
-	PasswordHash       string `gorm:"not null"`
+	PasswordHash        string `gorm:"not null"`
-	IsAdmin            bool   `gorm:"default:false"`
+	IsAdmin             bool   `gorm:"default:false"`
-	LastPasswordChange time.Time
+	LastPasswordChange  time.Time
-	FailedLoginAttempts int    `gorm:"default:0"`
+	FailedLoginAttempts int  `gorm:"default:0"`
-	AccountLocked      bool   `gorm:"default:false"`
+	AccountLocked       bool `gorm:"default:false"`
-	LockoutUntil       *time.Time
+	LockoutUntil        *time.Time
-	Theme              string `gorm:"default:'light'"`
+	Theme               string `gorm:"default:'light'"`
-	CreatedAt          time.Time
+	CreatedAt           time.Time
-	UpdatedAt          time.Time
+	UpdatedAt           time.Time
 }
 
 type PasswordHistory struct {
@@ -46,31 +46,31 @@ type PasswordResetToken struct {
 }
 
 type TransferConfig struct {
-	ID              uint   `gorm:"primarykey"`
+	ID             uint   `gorm:"primarykey"`
-	Name            string `gorm:"not null" form:"name"`
+	Name           string `gorm:"not null" form:"name"`
-	SourceType      string `gorm:"not null" form:"source_type"`
+	SourceType     string `gorm:"not null" form:"source_type"`
-	SourcePath      string `gorm:"not null" form:"source_path"`
+	SourcePath     string `gorm:"not null" form:"source_path"`
-	SourceHost      string `form:"source_host"`
+	SourceHost     string `form:"source_host"`
-	SourcePort      int    `gorm:"default:22" form:"source_port"`
+	SourcePort     int    `gorm:"default:22" form:"source_port"`
-	SourceUser      string `form:"source_user"`
+	SourceUser     string `form:"source_user"`
-	SourcePassword  string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
+	SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
-	SourceKeyFile   string `form:"source_key_file"`
+	SourceKeyFile  string `form:"source_key_file"`
 	// S3 source fields
-	SourceBucket     string `form:"source_bucket"`
+	SourceBucket    string `form:"source_bucket"`
-	SourceRegion     string `form:"source_region"`
+	SourceRegion    string `form:"source_region"`
-	SourceAccessKey  string `form:"source_access_key"`
+	SourceAccessKey string `form:"source_access_key"`
-	SourceSecretKey  string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
+	SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
-	SourceEndpoint   string `form:"source_endpoint"`
+	SourceEndpoint  string `form:"source_endpoint"`
 	// SMB source fields
-	SourceShare      string `form:"source_share"`
+	SourceShare  string `form:"source_share"`
-	SourceDomain     string `form:"source_domain"`
+	SourceDomain string `form:"source_domain"`
 	// FTP source fields
-	SourcePassiveMode bool   `gorm:"default:true" form:"source_passive_mode"`
+	SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
 	// OneDrive and Google Drive source fields
 	SourceClientID     string `form:"source_client_id"`
 	SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
-	SourceDriveID      string `form:"source_drive_id"`       // For OneDrive
+	SourceDriveID      string `form:"source_drive_id"`               // For OneDrive
-	SourceTeamDrive    string `form:"source_team_drive"`     // For Google Drive
+	SourceTeamDrive    string `form:"source_team_drive"`             // For Google Drive
 	// General fields
 	FilePattern     string `gorm:"default:'*'" form:"file_pattern"`
 	OutputPattern   string `form:"output_pattern"` // Pattern for output filenames with date variables
@@ -82,30 +82,30 @@ type TransferConfig struct {
 	DestPassword    string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
 	DestKeyFile     string `form:"dest_key_file"`
 	// S3 destination fields
-	DestBucket     string `form:"dest_bucket"`
+	DestBucket    string `form:"dest_bucket"`
-	DestRegion     string `form:"dest_region"`
+	DestRegion    string `form:"dest_region"`
-	DestAccessKey  string `form:"dest_access_key"`
+	DestAccessKey string `form:"dest_access_key"`
-	DestSecretKey  string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
+	DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
-	DestEndpoint   string `form:"dest_endpoint"`
+	DestEndpoint  string `form:"dest_endpoint"`
 	// SMB destination fields
-	DestShare      string `form:"dest_share"`
+	DestShare  string `form:"dest_share"`
-	DestDomain     string `form:"dest_domain"`
+	DestDomain string `form:"dest_domain"`
 	// FTP destination fields
-	DestPassiveMode bool   `gorm:"default:true" form:"dest_passive_mode"`
+	DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
 	// OneDrive and Google Drive destination fields
 	DestClientID     string `form:"dest_client_id"`
 	DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
-	DestDriveID      string `form:"dest_drive_id"`      // For OneDrive
+	DestDriveID      string `form:"dest_drive_id"`               // For OneDrive
-	DestTeamDrive    string `form:"dest_team_drive"`    // For Google Drive
+	DestTeamDrive    string `form:"dest_team_drive"`             // For Google Drive
 	// General fields
-	ArchivePath     string `form:"archive_path"`
+	ArchivePath         string `form:"archive_path"`
-	ArchiveEnabled  bool   `gorm:"default:false" form:"archive_enabled"`
+	ArchiveEnabled      bool   `gorm:"default:false" form:"archive_enabled"`
-	RcloneFlags     string `form:"rclone_flags"`
+	RcloneFlags         string `form:"rclone_flags"`
 	DeleteAfterTransfer bool   `gorm:"default:false" form:"delete_after_transfer"`
-	CreatedBy       uint
+	CreatedBy           uint
-	User            User `gorm:"foreignkey:CreatedBy"`
+	User                User `gorm:"foreignkey:CreatedBy"`
-	CreatedAt       time.Time
+	CreatedAt           time.Time
-	UpdatedAt       time.Time
+	UpdatedAt           time.Time
 }
 
 type Job struct {
@@ -135,6 +135,25 @@ type JobHistory struct {
 	ErrorMessage     string
 }
 
+// FileMetadata stores information about processed files
+type FileMetadata struct {
+	ID              uint   `gorm:"primarykey"`
+	JobID           uint   `gorm:"not null;index"`
+	Job             Job    `gorm:"foreignkey:JobID"`
+	FileName        string `gorm:"not null"`
+	OriginalPath    string `gorm:"not null"`
+	FileSize        int64  `gorm:"not null"`
+	FileHash        string `gorm:"index"` // MD5 or other hash for file identity
+	CreationTime    time.Time
+	ModTime         time.Time
+	ProcessedTime   time.Time `gorm:"not null"`
+	DestinationPath string    `gorm:"not null"`
+	Status          string    `gorm:"not null"` // processed, archived, deleted, etc.
+	ErrorMessage    string
+	CreatedAt       time.Time
+	UpdatedAt       time.Time
+}
+
 type DB struct {
 	*gorm.DB
 }
@@ -153,7 +172,7 @@ func Initialize(dbPath string) (*DB, error) {
 	}
 
 	// Auto migrate the schema
-	err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{})
+	err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{}, &FileMetadata{})
 	if err != nil {
 		return nil, fmt.Errorf("failed to migrate database: %v", err)
 	}
@@ -302,12 +321,63 @@ func (db *DB) UpdateJobHistory(history *JobHistory) error {
 }
 
 func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
-	var history []JobHistory
+	var histories []JobHistory
-	err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&history).Error
+	err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&histories).Error
-	return history, err
+	return histories, err
+}
+
+// CreateFileMetadata creates a new file metadata record
+func (db *DB) CreateFileMetadata(metadata *FileMetadata) error {
+	return db.Create(metadata).Error
+}
+
+// GetFileMetadata retrieves file metadata by ID
+func (db *DB) GetFileMetadata(id uint) (*FileMetadata, error) {
+	var metadata FileMetadata
+	err := db.First(&metadata, id).Error
+	if err != nil {
+		return nil, err
+	}
+	return &metadata, nil
+}
+
+// GetFileMetadataByJobAndName retrieves file metadata by job ID and filename
+func (db *DB) GetFileMetadataByJobAndName(jobID uint, fileName string) (*FileMetadata, error) {
+	var metadata FileMetadata
+	err := db.Where("job_id = ? AND file_name = ?", jobID, fileName).First(&metadata).Error
+	if err != nil {
+		return nil, err
+	}
+	return &metadata, nil
+}
+
+// GetFileMetadataByHash retrieves file metadata by file hash
+func (db *DB) GetFileMetadataByHash(fileHash string) (*FileMetadata, error) {
+	var metadata FileMetadata
+	err := db.Where("file_hash = ?", fileHash).First(&metadata).Error
+	if err != nil {
+		return nil, err
+	}
+	return &metadata, nil
+}
+
+// UpdateFileMetadata updates an existing file metadata record
+func (db *DB) UpdateFileMetadata(metadata *FileMetadata) error {
+	return db.Save(metadata).Error
+}
+
+// GetFileMetadataForJob retrieves all file metadata for a job
+func (db *DB) GetFileMetadataForJob(jobID uint) ([]FileMetadata, error) {
+	var metadata []FileMetadata
+	err := db.Where("job_id = ?", jobID).Find(&metadata).Error
+	return metadata, err
+}
+
+// DeleteFileMetadata deletes file metadata by ID
+func (db *DB) DeleteFileMetadata(id uint) error {
+	return db.Delete(&FileMetadata{}, id).Error
 }
 
-// Helper functions
 func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
 	return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
 }
@@ -345,7 +415,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 		if config.SourceKeyFile != "" {
 			args = append(args, "key_file", config.SourceKeyFile)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -362,11 +432,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.SourceEndpoint != "" {
 			args = append(args, "endpoint", config.SourceEndpoint)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -383,7 +453,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -397,7 +467,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -412,11 +482,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.SourceDomain != "" {
 			args = append(args, "domain", config.SourceDomain)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -431,11 +501,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.SourcePassiveMode {
 			args = append(args, "passive", "true")
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -450,7 +520,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -466,7 +536,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -480,11 +550,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.SourceDriveID != "" {
 			args = append(args, "drive_id", config.SourceDriveID)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -498,11 +568,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.SourceTeamDrive != "" {
 			args = append(args, "team_drive", config.SourceTeamDrive)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -533,7 +603,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 		if config.DestKeyFile != "" {
 			args = append(args, "key_file", config.DestKeyFile)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -550,11 +620,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.DestEndpoint != "" {
 			args = append(args, "endpoint", config.DestEndpoint)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -571,7 +641,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -585,7 +655,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -600,11 +670,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.DestDomain != "" {
 			args = append(args, "domain", config.DestDomain)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -619,11 +689,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.DestPassiveMode {
 			args = append(args, "passive", "true")
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -638,7 +708,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -654,7 +724,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -668,11 +738,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.DestDriveID != "" {
 			args = append(args, "drive_id", config.DestDriveID)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -686,11 +756,11 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
 			"--config", configPath,
 			"--log-level", "ERROR",
 		}
-		
+
 		if config.DestTeamDrive != "" {
 			args = append(args, "team_drive", config.DestTeamDrive)
 		}
-		
+
 		cmd := exec.Command(rclonePath, args...)
 		if output, err := cmd.CombinedOutput(); err != nil {
 			return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)

internal/scheduler/scheduler.go

@@ -1,12 +1,17 @@
 package scheduler
 
 import (
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -162,7 +167,7 @@ func (s *Scheduler) executeJob(jobID uint) {
 		"size",
 		"--include", job.Config.FilePattern,
 	}
-	
+
 	// Add source path with bucket for S3-compatible storage
 	var sourceSizePath string
 	if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -173,9 +178,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 	} else {
 		sourceSizePath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
 	}
-	
+
 	sizeArgs = append(sizeArgs, sourceSizePath)
-	
+
 	// Get the rclone path from the environment variable or use the default path
 	rclonePath := os.Getenv("RCLONE_PATH")
 	if rclonePath == "" {
@@ -200,7 +205,16 @@ func (s *Scheduler) executeJob(jobID uint) {
 
 	totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
 	totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
-	// totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
+	totalSize := strings.TrimSpace(strings.Split(outputStr, "Total size:")[1])
+	if strings.Contains(totalSize, "(") {
+		totalSize = strings.TrimSpace(strings.Split(totalSize, "(")[1])
+		totalSize = strings.TrimSpace(strings.Split(totalSize, " ")[0])
+	} else {
+		totalSize = "0"
+	}
+
+	bytesTransferred, _ := strconv.ParseInt(totalSize, 10, 64)
+	history.BytesTransferred = bytesTransferred
 
 	if totalObjects == "0" {
 		fmt.Printf("No files to transfer for job %d\n", jobID)
@@ -216,7 +230,7 @@ func (s *Scheduler) executeJob(jobID uint) {
 			"lsf",
 			"--include", job.Config.FilePattern,
 		}
-		
+
 		// Add source path with bucket for S3-compatible storage
 		var sourceLsPath string
 		if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -227,9 +241,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 		} else {
 			sourceLsPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
 		}
-		
+
 		listArgs = append(listArgs, sourceLsPath)
-		
+
 		fmt.Printf("Listing files for job %d: rclone %s\n", jobID, strings.Join(listArgs, " "))
 		// Get the rclone path from the environment variable or use the default path
 		rclonePath := os.Getenv("RCLONE_PATH")
@@ -238,29 +252,123 @@ func (s *Scheduler) executeJob(jobID uint) {
 		}
 		listCmd := exec.Command(rclonePath, listArgs...)
 		listOutput, listErr := listCmd.CombinedOutput()
-		
+
 		if listErr != nil {
 			fmt.Printf("Error listing files for job %d: %v\n", jobID, listErr)
 			history.Status = "failed"
 			history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
 			return
 		}
-		
+
 		// Split the output by newlines to get individual files
 		files := strings.Split(strings.TrimSpace(string(listOutput)), "\n")
 		fmt.Printf("Found %d files to transfer for job %d\n", len(files), jobID)
-		
+
 		var transferErrors []string
 		filesTransferred := 0
-		
+
 		// Process each file individually
 		for _, file := range files {
 			if file == "" {
 				continue
 			}
-			
+
 			fmt.Printf("Processing file: %s for job %d\n", file, jobID)
-			
+
+			// Get detailed file info if this is a local source
+			var fileSize int64
+			var createTime, modTime time.Time
+			var fileHash string
+			var fileMetadataErr error
+
+			// Get file metadata based on source type
+			if job.Config.SourceType == "local" {
+				// Construct the full local path
+				localFilePath := filepath.Join(job.Config.SourcePath, file)
+
+				// Get file info (size, creation time, modification time)
+				fileSize, createTime, modTime, fileMetadataErr = getFileInfo(localFilePath)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not get file info for %s: %v\n", file, fileMetadataErr)
+				}
+
+				// Calculate file hash (MD5)
+				fileHash, fileMetadataErr = calculateFileHash(localFilePath)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not calculate hash for %s: %v\n", file, fileMetadataErr)
+				}
+
+				// Check if this file has been processed before (by hash)
+				if fileHash != "" {
+					processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+					if processed {
+						fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
+							file, fileHash, prevMetadata.FileName)
+
+						// If configured to skip previously processed files,
+						// we could add that logic here
+						// For now, we'll just log it and continue
+					}
+				}
+
+				// Also check the processing history for this specific file name
+				prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+				if histErr == nil {
+					fmt.Printf("File %s was previously processed on %s with status: %s\n",
+						file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+
+					// If the file was previously processed successfully and the hash hasn't changed,
+					// we could skip processing
+					if prevMetadata.Status == "processed" ||
+						prevMetadata.Status == "archived" ||
+						prevMetadata.Status == "deleted" ||
+						prevMetadata.Status == "archived_and_deleted" {
+						if fileHash != "" && fileHash == prevMetadata.FileHash {
+							fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", file)
+							// Skip this file and continue to the next one
+							continue
+						}
+					}
+				}
+			} else {
+				// For non-local sources, use rclone lsjson to get metadata
+				fileSize, createTime, modTime, fileHash, fileMetadataErr = s.getRemoteFileInfo(&job.Config, file)
+				if fileMetadataErr != nil {
+					fmt.Printf("Warning: Could not get remote file info for %s: %v\n", file, fileMetadataErr)
+					// We'll continue with placeholder values
+					fileSize = 0
+					createTime = time.Now()
+					modTime = time.Now()
+					fileHash = ""
+				} else {
+					// If we got a hash, check for previous processing
+					if fileHash != "" {
+						processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
+						if processed {
+							fmt.Printf("Remote file %s has been processed before (hash: %s, previous file: %s)\n",
+								file, fileHash, prevMetadata.FileName)
+
+							// Skip previously processed files with the same hash if they were processed successfully
+							if prevMetadata.Status == "processed" ||
+								prevMetadata.Status == "archived" ||
+								prevMetadata.Status == "deleted" ||
+								prevMetadata.Status == "archived_and_deleted" {
+								fmt.Printf("Skipping unchanged remote file %s (hash matches previous processing)\n", file)
+								// Skip this file and continue to the next one
+								continue
+							}
+						}
+					}
+
+					// Also check by filename
+					prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
+					if histErr == nil {
+						fmt.Printf("Remote file %s was previously processed on %s with status: %s\n",
+							file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
+					}
+				}
+			}
+
 			// Prepare moveto command for transfer
 			transferArgs := []string{
 				"--config", configPath,
@@ -270,10 +378,10 @@ func (s *Scheduler) executeJob(jobID uint) {
 				"--verbose",
 				"--stats", "1s",
 			}
-			
+
 			// Source and destination paths
 			var sourcePath, destPath string
-			
+
 			// For S3, MinIO, and B2, include the bucket in the path
 			if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
 				sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, file)
@@ -283,7 +391,9 @@ func (s *Scheduler) executeJob(jobID uint) {
 			} else {
 				sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
 			}
-			
+
+			var destFile string = file
+
 			if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
 				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
 				if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
@@ -292,27 +402,36 @@ func (s *Scheduler) executeJob(jobID uint) {
 			} else {
 				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, file)
 			}
-			
+
 			// Add output filename pattern if specified
 			if job.Config.OutputPattern != "" {
 				// Process the output pattern for this specific file
-				newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
+				destFile = ProcessOutputPattern(job.Config.OutputPattern, file)
-				destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
+
-				fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
+				if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+					destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
+					if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+						destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
+					}
+				} else {
+					destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
+				}
+
+				fmt.Printf("Renaming file from %s to %s for job %d\n", file, destFile, jobID)
 			}
-			
+
 			// Add custom flags if specified
 			if job.Config.RcloneFlags != "" {
 				customFlags := strings.Split(job.Config.RcloneFlags, " ")
 				transferArgs = append(transferArgs, customFlags...)
 				fmt.Printf("Added custom flags for job %d: %v\n", jobID, customFlags)
 			}
-			
+
 			// Add source and destination to the command
 			transferArgs = append(transferArgs, sourcePath, destPath)
-			
+
 			// Execute transfer for this file
-			fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n", 
+			fmt.Printf("Executing rclone transfer command for job %d, file %s: rclone %s\n",
 				jobID, file, strings.Join(transferArgs, " "))
 			// Get the rclone path from the environment variable or use the default path
 			rclonePath := os.Getenv("RCLONE_PATH")
@@ -325,25 +444,48 @@ func (s *Scheduler) executeJob(jobID uint) {
 			// Print the output
 			fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
 
+			// Create file metadata record
+			fileStatus := "processed"
+			var fileErrorMsg string
+			var destPathForDB string
+
 			// Check if file was successfully transferred
 			if fileErr != nil {
 				fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
 				transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
+				fileStatus = "error"
+				fileErrorMsg = fileErr.Error()
 			} else {
 				filesTransferred++
 				fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
-				
+
+				// Extract the actual destination path (without rclone remote prefix)
+				if job.Config.DestinationType == "local" {
+					destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
+				} else {
+					// For remote destinations, store the path format
+					if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
+						if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
+							destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
+						} else {
+							destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
+						}
+					} else {
+						destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
+					}
+				}
+
 				// If archiving is enabled and transfer was successful, move files to archive
 				if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
 					fmt.Printf("Archiving file %s for job %d\n", file, jobID)
-					
+
 					// We don't need to move the file since we used moveto, but we can copy it to archive
 					archiveArgs := []string{
 						"--config", configPath,
 						"copyto",
 						sourcePath,
 					}
-					
+
 					// Construct archive path with bucket if needed
 					var archiveDest string
 					if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
@@ -351,10 +493,10 @@ func (s *Scheduler) executeJob(jobID uint) {
 					} else {
 						archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, file)
 					}
-					
+
 					archiveArgs = append(archiveArgs, archiveDest)
-					
+
-					fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n", 
+					fmt.Printf("Executing rclone archive command for job %d, file %s: rclone %s\n",
 						jobID, file, strings.Join(archiveArgs, " "))
 					// Get the rclone path from the environment variable or use the default path
 					rclonePath := os.Getenv("RCLONE_PATH")
@@ -370,34 +512,64 @@ func (s *Scheduler) executeJob(jobID uint) {
 					// Check if file was successfully transferred
 					if archiveErr != nil {
 						fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
-						transferErrors = append(transferErrors, 
+						transferErrors = append(transferErrors,
 							fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
+					} else {
+						fileStatus = "archived"
 					}
 				}
+
 				if job.Config.DeleteAfterTransfer {
 					fmt.Printf("Deleting file %s for job %d\n", file, jobID)
 					deleteArgs := []string{
 						"--config", configPath,
 						"deletefile",
-						sourcePath,					}
+						sourcePath}
 					deleteCmd := exec.Command(rclonePath, deleteArgs...)
 					deleteOutput, deleteErr := deleteCmd.CombinedOutput()
 					fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
 					if deleteErr != nil {
 						fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
-						transferErrors = append(transferErrors, 
+						transferErrors = append(transferErrors,
 							fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
+					} else {
+						if fileStatus == "archived" {
+							fileStatus = "archived_and_deleted"
+						} else {
+							fileStatus = "deleted"
+						}
 					}
 				}
 			}
+
+			// Create and save file metadata
+			metadata := &db.FileMetadata{
+				JobID:           jobID,
+				FileName:        file,
+				OriginalPath:    job.Config.SourcePath,
+				FileSize:        fileSize,
+				FileHash:        fileHash,
+				CreationTime:    createTime,
+				ModTime:         modTime,
+				ProcessedTime:   time.Now(),
+				DestinationPath: destPathForDB,
+				Status:          fileStatus,
+				ErrorMessage:    fileErrorMsg,
+			}
+
+			if err := s.db.CreateFileMetadata(metadata); err != nil {
+				fmt.Printf("Error creating file metadata for %s: %v\n", file, err)
+			} else {
+				fmt.Printf("Created file metadata record for %s (ID: %d)\n", file, metadata.ID)
+			}
 		}
-		
+
 		// Update job history with transfer results
 		history.FilesTransferred = filesTransferred
-		
+
 		if len(transferErrors) > 0 {
 			history.Status = "completed_with_errors"
-			history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s", 
+			history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
 				len(transferErrors), strings.Join(transferErrors, "\n"))
 		}
 	}
@@ -439,15 +611,15 @@ func ProcessOutputPattern(pattern string, originalFilename string) string {
 		format := dateRegex.FindStringSubmatch(match)[1]
 		return time.Now().Format(format)
 	})
-	
+
 	// Split the filename and extension
 	ext := filepath.Ext(originalFilename)
 	filename := strings.TrimSuffix(originalFilename, ext)
-	
+
 	// Replace filename and extension variables
 	processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
 	processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
-	
+
 	return processedPattern
 }
 
@@ -467,31 +639,31 @@ func createRcloneFilterFile(pattern string) (string, error) {
 		format := dateRegex.FindStringSubmatch(match)[1]
 		return time.Now().Format(format)
 	})
-	
+
 	// Replace filename and extension variables with rclone's capture group references
 	// For rclone rename filters, we need to use {1} for the first capture group, not $1
 	// See: https://rclone.org/filtering/#rename
-	
+
 	// Extract filename without extension
 	processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
-	
+
 	// Extract extension (with the dot)
 	processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
-	
+
 	// Create a rename rule for rclone using the correct syntax:
 	// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
 	// - For files with extension: capture the name and extension separately
 	rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
-	
+
 	// Add a fallback rule for files without extension
-	fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n", 
+	fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
 		strings.ReplaceAll(processedPattern, "{2}", ""))
-	
+
 	// Write the rules to the file
 	if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
 		return "", fmt.Errorf("failed to write to filter file: %v", err)
 	}
-	
+
 	return tmpFile.Name(), nil
 }
 
@@ -515,3 +687,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
 	go s.executeJob(jobID)
 	return nil
 }
+
+// calculateFileHash computes an MD5 hash for the given file path
+func calculateFileHash(filePath string) (string, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", fmt.Errorf("error opening file: %v", err)
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	if _, err := io.Copy(hash, file); err != nil {
+		return "", fmt.Errorf("error calculating hash: %v", err)
+	}
+
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
+
+// getFileInfo retrieves file stats like size, creation time, and modification time
+func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
+	info, err := os.Stat(filePath)
+	if err != nil {
+		return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
+	}
+
+	size := info.Size()
+	modTime := info.ModTime()
+
+	// Get creation time (this is platform-specific)
+	// For simplicity, we'll use modification time as a fallback
+	createTime := modTime
+
+	return size, createTime, modTime, nil
+}
+
+// hasFileBeenProcessed checks if a file with the same hash has been processed before
+func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
+	if fileHash == "" {
+		return false, nil, nil
+	}
+
+	// First try to find by hash (most reliable)
+	metadata, err := s.db.GetFileMetadataByHash(fileHash)
+	if err == nil && metadata != nil {
+		return true, metadata, nil
+	}
+
+	return false, nil, nil
+}
+
+// checkFileProcessingHistory checks processing history for a given file
+func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
+	// Try to find by job and filename
+	metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
+	if err == nil && metadata != nil {
+		return metadata, nil
+	}
+
+	return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
+}
+
+// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
+func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
+	// Get rclone config path
+	configPath := s.db.GetConfigRclonePath(config)
+
+	// Construct the appropriate source path
+	var sourcePath string
+	if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
+		if config.SourcePath != "" && config.SourcePath != "/" {
+			sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
+		}
+	} else {
+		sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
+	}
+
+	// Use rclone lsjson to get file details
+	rclonePath := os.Getenv("RCLONE_PATH")
+	if rclonePath == "" {
+		rclonePath = "rclone"
+	}
+
+	// Construct the full path to the file
+	fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
+
+	// Run rclone lsjson command
+	args := []string{
+		"--config", configPath,
+		"lsjson",
+		"--hash",
+		fullPath,
+	}
+
+	cmd := exec.Command(rclonePath, args...)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
+	}
+
+	// Parse the JSON output
+	var files []map[string]interface{}
+	if err := json.Unmarshal(output, &files); err != nil {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
+	}
+
+	if len(files) == 0 {
+		return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
+	}
+
+	fileInfo := files[0]
+
+	// Extract file size
+	var fileSize int64
+	if size, ok := fileInfo["Size"].(float64); ok {
+		fileSize = int64(size)
+	}
+
+	// Extract modification time
+	modTime := time.Now()
+	if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
+		if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
+			modTime = parsedTime
+		}
+	}
+
+	// Create time is usually not available for remote files, so we'll use modTime
+	createTime := modTime
+
+	// Calculate hash if available
+	var md5Hash string
+	if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
+		if md5, ok := hashes["md5"].(string); ok {
+			md5Hash = md5
+		}
+	}
+
+	return fileSize, createTime, modTime, md5Hash, nil
+}

internal/web/handlers/file_metadata_handlers.go

@@ -0,0 +1,378 @@
+package handlers
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/starfleetcptn/gomft/components"
+	"github.com/starfleetcptn/gomft/internal/db"
+)
+
+// FileMetadataHandler handles displaying and searching file metadata
+type FileMetadataHandler struct {
+	DB *db.DB
+}
+
+// Register registers the file metadata routes
+func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
+	fileGroup := router.Group("/files")
+
+	fileGroup.GET("", h.ListFileMetadata)
+	fileGroup.GET("/:id", h.GetFileMetadataDetails)
+	fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
+	fileGroup.GET("/search", h.SearchFileMetadata)
+	fileGroup.DELETE("/:id", h.DeleteFileMetadata)
+}
+
+// ListFileMetadata displays a list of file metadata with pagination and filtering options
+func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Query parameters for pagination and filtering
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	jobIDStr := c.Query("job_id")
+	fileName := c.Query("filename")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+	// Apply filters
+	if jobIDStr != "" {
+		jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+		query = query.Where("file_metadata.job_id = ?", jobID)
+	} else {
+		// Only show files from jobs created by the current user
+		query = query.Where("jobs.created_by = ?", userID)
+	}
+
+	if status != "" {
+		query = query.Where("file_metadata.status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err := query.Preload("Job").Preload("Job.Config").
+		Order("file_metadata.processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata list template
+	data := components.FileMetadataListData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Filter: components.FileMetadataFilter{
+			Status:   status,
+			JobID:    jobIDStr,
+			FileName: fileName,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataDetails displays detailed information about a specific file
+func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get file ID from URL parameter
+	fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+		return
+	}
+
+	// Retrieve file metadata
+	var fileMetadata db.FileMetadata
+	err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+		return
+	}
+
+	// Check if the user has access to this file (file must belong to a job created by the user)
+	var jobCreator uint
+	err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+	if err != nil || jobCreator != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata details template
+	data := components.FileMetadataDetailsData{
+		File: fileMetadata,
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataForJob displays file metadata for a specific job
+func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get job ID from URL parameter
+	jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
+		return
+	}
+
+	// Check if the user has access to this job
+	var job db.Job
+	err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
+		return
+	}
+
+	if job.CreatedBy != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
+		return
+	}
+
+	// Query parameters for pagination
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	fileName := c.Query("filename")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
+
+	// Apply filters
+	if status != "" {
+		query = query.Where("status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err = query.Preload("Job").Preload("Job.Config").
+		Order("processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata list template
+	data := components.FileMetadataListData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Job:        &job,
+		Filter: components.FileMetadataFilter{
+			Status:   status,
+			JobID:    strconv.FormatUint(uint64(job.ID), 10),
+			FileName: fileName,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// SearchFileMetadata searches file metadata based on various criteria
+func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Query parameters for search and pagination
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	jobIDStr := c.Query("job_id")
+	fileName := c.Query("filename")
+	hash := c.Query("hash")
+	startDate := c.Query("start_date")
+	endDate := c.Query("end_date")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+	// Apply filters
+	if jobIDStr != "" {
+		jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+		query = query.Where("file_metadata.job_id = ?", jobID)
+	} else {
+		// Only show files from jobs created by the current user
+		query = query.Where("jobs.created_by = ?", userID)
+	}
+
+	if status != "" {
+		query = query.Where("file_metadata.status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	if hash != "" {
+		query = query.Where("file_metadata.file_hash = ?", hash)
+	}
+
+	if startDate != "" {
+		query = query.Where("file_metadata.processed_time >= ?", startDate)
+	}
+
+	if endDate != "" {
+		query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err := query.Preload("Job").Preload("Job.Config").
+		Order("file_metadata.processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata search template
+	data := components.FileMetadataSearchData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Filter: components.FileMetadataFilter{
+			Status:    status,
+			JobID:     jobIDStr,
+			FileName:  fileName,
+			Hash:      hash,
+			StartDate: startDate,
+			EndDate:   endDate,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
+}
+
+// DeleteFileMetadata deletes a file metadata record
+func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get file ID from URL parameter
+	fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+		return
+	}
+
+	// Check if the user has access to this file
+	var fileMetadata db.FileMetadata
+	err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+		return
+	}
+
+	var jobCreator uint
+	err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+	if err != nil || jobCreator != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
+		return
+	}
+
+	// Delete the file metadata
+	err = h.DB.DeleteFileMetadata(uint(fileID))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
+		return
+	}
+
+	// Redirect to the file list
+	c.Redirect(http.StatusFound, "/files")
+}

internal/web/handlers/routes.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"strconv"
 	"net/url"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	"github.com/starfleetcptn/gomft/components"
@@ -15,13 +15,13 @@ import (
 // HandleHistory handles the GET /history route
 func (h *Handlers) HandleHistory(c *gin.Context) {
 	userID := c.GetUint("userID")
-	
+
 	// Get pagination parameters
 	page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
 	if err != nil || page < 1 {
 		page = 1
 	}
-	
+
 	pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
 	if err != nil {
 		pageSize = 10
@@ -30,47 +30,47 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
 	if pageSize != 10 && pageSize != 25 && pageSize != 50 && pageSize != 100 {
 		pageSize = 10
 	}
-	
+
 	// Get search term
 	searchTerm := c.Query("search")
-	
+
 	// Build the query
 	query := h.DB.Model(&db.JobHistory{}).
 		Joins("JOIN jobs ON jobs.id = job_histories.job_id").
 		Joins("JOIN transfer_configs ON transfer_configs.id = jobs.config_id").
 		Where("jobs.created_by = ?", userID)
-	
+
 	// Apply search if provided
 	if searchTerm != "" {
-		query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?", 
+		query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
 			"%"+searchTerm+"%", "%"+searchTerm+"%")
 	}
-	
+
 	// Count total matching records for pagination
 	var total int64
 	query.Count(&total)
-	
+
 	// Calculate total pages
 	totalPages := int(math.Ceil(float64(total) / float64(pageSize)))
 	if totalPages == 0 {
 		totalPages = 1
 	}
-	
+
 	// Ensure page is within bounds
 	if page > totalPages {
 		page = totalPages
 	}
-	
+
 	// Get paginated results
 	var history []db.JobHistory
 	offset := (page - 1) * pageSize
-	
+
 	query.Offset(offset).
 		Limit(pageSize).
 		Preload("Job.Config").
 		Order("start_time desc").
 		Find(&history)
-	
+
 	// If we got no results and we're not on page 1, redirect to page 1
 	// Only do this for non-HTMX requests to avoid navigation issues
 	isHtmxRequest := c.GetHeader("HX-Request") == "true"
@@ -82,7 +82,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
 		c.Redirect(http.StatusFound, redirectURL)
 		return
 	}
-	
+
 	data := components.HistoryData{
 		History:     history,
 		CurrentPage: page,
@@ -160,10 +160,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 	// Protected routes
 	authorized := router.Group("/")
 	authorized.Use(h.AuthMiddleware())
-	
+
 	// Password change route - only accessed from profile page
 	authorized.POST("/change-password", h.HandleChangePassword)
-	
+
 	{
 		authorized.GET("/dashboard", h.HandleDashboard)
 		authorized.GET("/configs", h.HandleConfigs)
@@ -186,12 +186,16 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		authorized.GET("/profile", h.HandleProfile)
 		authorized.POST("/profile/theme", h.HandleUpdateTheme)
 		authorized.POST("/logout", h.HandleLogout)
-		
+
+		// File metadata routes
+		fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
+		fileMetadataHandler.Register(authorized)
+
 		// AJAX routes for dashboard
 		authorized.GET("/dashboard/data", h.HandleDashboardData)
 		authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
 		authorized.GET("/dashboard/history", h.HandleDashboardHistoryData)
-		
+
 		// Test connection routes
 		authorized.POST("/test-connection", h.HandleTestConnection)
 		authorized.POST("/test-sftp-connection", h.HandleTestSFTPConnection)
@@ -208,7 +212,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		admin.DELETE("/users/:id", h.HandleDeleteUser)
 		admin.GET("/register", h.HandleRegisterPage)
 		admin.POST("/register", h.HandleRegister)
-		
+
 		// Admin tools routes
 		admin.GET("/tools", h.HandleAdminTools)
 		admin.POST("/backup-database", h.HandleBackupDatabase)
@@ -222,12 +226,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		admin.DELETE("/delete-backup/:filename", h.HandleDeleteBackup)
 		admin.GET("/refresh-backups", h.HandleRefreshBackups)
 	}
-	
+
 	// API routes
 	api := router.Group("/api")
 	{
 		api.POST("/login", h.HandleAPILogin)
-		
+
 		// Protected API routes
 		apiAuthorized := api.Group("/")
 		apiAuthorized.Use(h.APIAuthMiddleware())
@@ -238,7 +242,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 			apiAuthorized.POST("/configs", h.HandleAPICreateConfig)
 			apiAuthorized.PUT("/configs/:id", h.HandleAPIUpdateConfig)
 			apiAuthorized.DELETE("/configs/:id", h.HandleAPIDeleteConfig)
-			
+
 			// Job endpoints
 			apiAuthorized.GET("/jobs", h.HandleAPIJobs)
 			apiAuthorized.GET("/jobs/:id", h.HandleAPIJob)
@@ -246,11 +250,11 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 			apiAuthorized.PUT("/jobs/:id", h.HandleAPIUpdateJob)
 			apiAuthorized.DELETE("/jobs/:id", h.HandleAPIDeleteJob)
 			apiAuthorized.POST("/jobs/:id/run", h.HandleAPIRunJob)
-			
+
 			// History endpoints
 			apiAuthorized.GET("/history", h.HandleAPIHistory)
 			apiAuthorized.GET("/job-runs/:id", h.HandleAPIJobRun)
-			
+
 			// Admin-only API routes
 			apiAdmin := apiAuthorized.Group("/admin")
 			apiAdmin.Use(h.APIAdminMiddleware())

internal/web/middleware/auth.go

@@ -1,3 +1,4 @@
+
 // AuthMiddleware is a middleware function that checks if the request has a valid JWT token
 func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
@@ -41,4 +42,4 @@ func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
 
 		c.Next()
 	}
-} 
+}