feat: Add file metadata tracking and search functionality

- Implement FileMetadata model to track processed files
- Create file metadata handlers for listing, searching, and viewing files
- Add file metadata routes and UI components
- Support advanced file search with multiple filters
- Enhance job execution to capture file metadata during transfers
- Implement file hash and duplicate detection logic

internal/web/handlers/file_metadata_handlers.go

@@ -0,0 +1,378 @@
+package handlers
+
+import (
+	"net/http"
+	"strconv"
+
+	"github.com/gin-gonic/gin"
+	"github.com/starfleetcptn/gomft/components"
+	"github.com/starfleetcptn/gomft/internal/db"
+)
+
+// FileMetadataHandler handles displaying and searching file metadata
+type FileMetadataHandler struct {
+	DB *db.DB
+}
+
+// Register registers the file metadata routes
+func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
+	fileGroup := router.Group("/files")
+
+	fileGroup.GET("", h.ListFileMetadata)
+	fileGroup.GET("/:id", h.GetFileMetadataDetails)
+	fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
+	fileGroup.GET("/search", h.SearchFileMetadata)
+	fileGroup.DELETE("/:id", h.DeleteFileMetadata)
+}
+
+// ListFileMetadata displays a list of file metadata with pagination and filtering options
+func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Query parameters for pagination and filtering
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	jobIDStr := c.Query("job_id")
+	fileName := c.Query("filename")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+	// Apply filters
+	if jobIDStr != "" {
+		jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+		query = query.Where("file_metadata.job_id = ?", jobID)
+	} else {
+		// Only show files from jobs created by the current user
+		query = query.Where("jobs.created_by = ?", userID)
+	}
+
+	if status != "" {
+		query = query.Where("file_metadata.status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err := query.Preload("Job").Preload("Job.Config").
+		Order("file_metadata.processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata list template
+	data := components.FileMetadataListData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Filter: components.FileMetadataFilter{
+			Status:   status,
+			JobID:    jobIDStr,
+			FileName: fileName,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataDetails displays detailed information about a specific file
+func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get file ID from URL parameter
+	fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+		return
+	}
+
+	// Retrieve file metadata
+	var fileMetadata db.FileMetadata
+	err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+		return
+	}
+
+	// Check if the user has access to this file (file must belong to a job created by the user)
+	var jobCreator uint
+	err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+	if err != nil || jobCreator != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata details template
+	data := components.FileMetadataDetailsData{
+		File: fileMetadata,
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
+}
+
+// GetFileMetadataForJob displays file metadata for a specific job
+func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get job ID from URL parameter
+	jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
+		return
+	}
+
+	// Check if the user has access to this job
+	var job db.Job
+	err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
+		return
+	}
+
+	if job.CreatedBy != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
+		return
+	}
+
+	// Query parameters for pagination
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	fileName := c.Query("filename")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
+
+	// Apply filters
+	if status != "" {
+		query = query.Where("status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err = query.Preload("Job").Preload("Job.Config").
+		Order("processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata list template
+	data := components.FileMetadataListData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Job:        &job,
+		Filter: components.FileMetadataFilter{
+			Status:   status,
+			JobID:    strconv.FormatUint(uint64(job.ID), 10),
+			FileName: fileName,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
+}
+
+// SearchFileMetadata searches file metadata based on various criteria
+func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Query parameters for search and pagination
+	page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
+	if page < 1 {
+		page = 1
+	}
+
+	limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
+	if limit < 1 || limit > 100 {
+		limit = 50
+	}
+
+	status := c.Query("status")
+	jobIDStr := c.Query("job_id")
+	fileName := c.Query("filename")
+	hash := c.Query("hash")
+	startDate := c.Query("start_date")
+	endDate := c.Query("end_date")
+
+	// Base query
+	query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
+
+	// Apply filters
+	if jobIDStr != "" {
+		jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
+		query = query.Where("file_metadata.job_id = ?", jobID)
+	} else {
+		// Only show files from jobs created by the current user
+		query = query.Where("jobs.created_by = ?", userID)
+	}
+
+	if status != "" {
+		query = query.Where("file_metadata.status = ?", status)
+	}
+
+	if fileName != "" {
+		query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
+	}
+
+	if hash != "" {
+		query = query.Where("file_metadata.file_hash = ?", hash)
+	}
+
+	if startDate != "" {
+		query = query.Where("file_metadata.processed_time >= ?", startDate)
+	}
+
+	if endDate != "" {
+		query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
+	}
+
+	// Count total records for pagination
+	var totalCount int64
+	query.Count(&totalCount)
+
+	// Retrieve file metadata with pagination
+	var fileMetadata []db.FileMetadata
+	offset := (page - 1) * limit
+	err := query.Preload("Job").Preload("Job.Config").
+		Order("file_metadata.processed_time DESC").
+		Offset(offset).Limit(limit).
+		Find(&fileMetadata).Error
+
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
+		return
+	}
+
+	// Create context for template
+	ctx := components.CreateTemplateContext(c)
+
+	// Render the file metadata search template
+	data := components.FileMetadataSearchData{
+		Files:      fileMetadata,
+		TotalCount: totalCount,
+		Page:       page,
+		Limit:      limit,
+		TotalPages: int(totalCount) / limit,
+		Filter: components.FileMetadataFilter{
+			Status:    status,
+			JobID:     jobIDStr,
+			FileName:  fileName,
+			Hash:      hash,
+			StartDate: startDate,
+			EndDate:   endDate,
+		},
+	}
+
+	// If total count is not exactly divisible by limit, add one more page
+	if int(totalCount)%limit > 0 {
+		data.TotalPages++
+	}
+
+	c.Header("Content-Type", "text/html")
+	components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
+}
+
+// DeleteFileMetadata deletes a file metadata record
+func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
+	userID := c.GetUint("userID")
+
+	// Get file ID from URL parameter
+	fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
+	if err != nil {
+		c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
+		return
+	}
+
+	// Check if the user has access to this file
+	var fileMetadata db.FileMetadata
+	err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
+	if err != nil {
+		c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
+		return
+	}
+
+	var jobCreator uint
+	err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
+	if err != nil || jobCreator != userID {
+		c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
+		return
+	}
+
+	// Delete the file metadata
+	err = h.DB.DeleteFileMetadata(uint(fileID))
+	if err != nil {
+		c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
+		return
+	}
+
+	// Redirect to the file list
+	c.Redirect(http.StatusFound, "/files")
+}

internal/web/handlers/routes.go

@@ -4,8 +4,8 @@ import (
 	"fmt"
 	"math"
 	"net/http"
-	"strconv"
 	"net/url"
+	"strconv"
 
 	"github.com/gin-gonic/gin"
 	"github.com/starfleetcptn/gomft/components"
@@ -15,13 +15,13 @@ import (
 // HandleHistory handles the GET /history route
 func (h *Handlers) HandleHistory(c *gin.Context) {
 	userID := c.GetUint("userID")
-	
+
 	// Get pagination parameters
 	page, err := strconv.Atoi(c.DefaultQuery("page", "1"))
 	if err != nil || page < 1 {
 		page = 1
 	}
-	
+
 	pageSize, err := strconv.Atoi(c.DefaultQuery("pageSize", "10"))
 	if err != nil {
 		pageSize = 10
@@ -30,47 +30,47 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
 	if pageSize != 10 && pageSize != 25 && pageSize != 50 && pageSize != 100 {
 		pageSize = 10
 	}
-	
+
 	// Get search term
 	searchTerm := c.Query("search")
-	
+
 	// Build the query
 	query := h.DB.Model(&db.JobHistory{}).
 		Joins("JOIN jobs ON jobs.id = job_histories.job_id").
 		Joins("JOIN transfer_configs ON transfer_configs.id = jobs.config_id").
 		Where("jobs.created_by = ?", userID)
-	
+
 	// Apply search if provided
 	if searchTerm != "" {
-		query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?", 
+		query = query.Where("transfer_configs.name LIKE ? OR job_histories.status LIKE ?",
 			"%"+searchTerm+"%", "%"+searchTerm+"%")
 	}
-	
+
 	// Count total matching records for pagination
 	var total int64
 	query.Count(&total)
-	
+
 	// Calculate total pages
 	totalPages := int(math.Ceil(float64(total) / float64(pageSize)))
 	if totalPages == 0 {
 		totalPages = 1
 	}
-	
+
 	// Ensure page is within bounds
 	if page > totalPages {
 		page = totalPages
 	}
-	
+
 	// Get paginated results
 	var history []db.JobHistory
 	offset := (page - 1) * pageSize
-	
+
 	query.Offset(offset).
 		Limit(pageSize).
 		Preload("Job.Config").
 		Order("start_time desc").
 		Find(&history)
-	
+
 	// If we got no results and we're not on page 1, redirect to page 1
 	// Only do this for non-HTMX requests to avoid navigation issues
 	isHtmxRequest := c.GetHeader("HX-Request") == "true"
@@ -82,7 +82,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
 		c.Redirect(http.StatusFound, redirectURL)
 		return
 	}
-	
+
 	data := components.HistoryData{
 		History:     history,
 		CurrentPage: page,
@@ -160,10 +160,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 	// Protected routes
 	authorized := router.Group("/")
 	authorized.Use(h.AuthMiddleware())
-	
+
 	// Password change route - only accessed from profile page
 	authorized.POST("/change-password", h.HandleChangePassword)
-	
+
 	{
 		authorized.GET("/dashboard", h.HandleDashboard)
 		authorized.GET("/configs", h.HandleConfigs)
@@ -186,12 +186,16 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		authorized.GET("/profile", h.HandleProfile)
 		authorized.POST("/profile/theme", h.HandleUpdateTheme)
 		authorized.POST("/logout", h.HandleLogout)
-		
+
+		// File metadata routes
+		fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
+		fileMetadataHandler.Register(authorized)
+
 		// AJAX routes for dashboard
 		authorized.GET("/dashboard/data", h.HandleDashboardData)
 		authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
 		authorized.GET("/dashboard/history", h.HandleDashboardHistoryData)
-		
+
 		// Test connection routes
 		authorized.POST("/test-connection", h.HandleTestConnection)
 		authorized.POST("/test-sftp-connection", h.HandleTestSFTPConnection)
@@ -208,7 +212,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		admin.DELETE("/users/:id", h.HandleDeleteUser)
 		admin.GET("/register", h.HandleRegisterPage)
 		admin.POST("/register", h.HandleRegister)
-		
+
 		// Admin tools routes
 		admin.GET("/tools", h.HandleAdminTools)
 		admin.POST("/backup-database", h.HandleBackupDatabase)
@@ -222,12 +226,12 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 		admin.DELETE("/delete-backup/:filename", h.HandleDeleteBackup)
 		admin.GET("/refresh-backups", h.HandleRefreshBackups)
 	}
-	
+
 	// API routes
 	api := router.Group("/api")
 	{
 		api.POST("/login", h.HandleAPILogin)
-		
+
 		// Protected API routes
 		apiAuthorized := api.Group("/")
 		apiAuthorized.Use(h.APIAuthMiddleware())
@@ -238,7 +242,7 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 			apiAuthorized.POST("/configs", h.HandleAPICreateConfig)
 			apiAuthorized.PUT("/configs/:id", h.HandleAPIUpdateConfig)
 			apiAuthorized.DELETE("/configs/:id", h.HandleAPIDeleteConfig)
-			
+
 			// Job endpoints
 			apiAuthorized.GET("/jobs", h.HandleAPIJobs)
 			apiAuthorized.GET("/jobs/:id", h.HandleAPIJob)
@@ -246,11 +250,11 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
 			apiAuthorized.PUT("/jobs/:id", h.HandleAPIUpdateJob)
 			apiAuthorized.DELETE("/jobs/:id", h.HandleAPIDeleteJob)
 			apiAuthorized.POST("/jobs/:id/run", h.HandleAPIRunJob)
-			
+
 			// History endpoints
 			apiAuthorized.GET("/history", h.HandleAPIHistory)
 			apiAuthorized.GET("/job-runs/:id", h.HandleAPIJobRun)
-			
+
 			// Admin-only API routes
 			apiAdmin := apiAuthorized.Group("/admin")
 			apiAdmin.Use(h.APIAdminMiddleware())

internal/web/middleware/auth.go

@@ -1,3 +1,4 @@
+
 // AuthMiddleware is a middleware function that checks if the request has a valid JWT token
 func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
 	return func(c *gin.Context) {
@@ -41,4 +42,4 @@ func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
 
 		c.Next()
 	}
-} 
+}