From fcf6ee25a8e6247cdc8d02849a11955bf67eb00a Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sun, 9 Mar 2025 10:22:57 -0700 Subject: [PATCH] feat: Add file metadata tracking and search functionality - Implement FileMetadata model to track processed files - Create file metadata handlers for listing, searching, and viewing files - Add file metadata routes and UI components - Support advanced file search with multiple filters - Enhance job execution to capture file metadata during transfers - Implement file hash and duplicate detection logic --- components/file_metadata.templ | 587 ++++++++++++++++++ components/layout.templ | 10 + components/providers/common/common.templ | 88 +-- components/providers/destination/ftp.templ | 1 + components/providers/source/ftp.templ | 1 + internal/db/db.go | 236 ++++--- internal/scheduler/scheduler.go | 404 ++++++++++-- .../web/handlers/file_metadata_handlers.go | 378 +++++++++++ internal/web/handlers/routes.go | 52 +- internal/web/middleware/auth.go | 3 +- 10 files changed, 1570 insertions(+), 190 deletions(-) create mode 100644 components/file_metadata.templ create mode 100644 internal/web/handlers/file_metadata_handlers.go diff --git a/components/file_metadata.templ b/components/file_metadata.templ new file mode 100644 index 0000000..d8dbc31 --- /dev/null +++ b/components/file_metadata.templ @@ -0,0 +1,587 @@ +package components + +import ( + "context" + "strconv" + "fmt" + "github.com/starfleetcptn/gomft/internal/db" +) + +// FileMetadataFilter represents filter parameters for file metadata queries +type FileMetadataFilter struct { + Status string + JobID string + FileName string + Hash string + StartDate string + EndDate string +} + +// FileMetadataListData contains data for the file metadata list template +type FileMetadataListData struct { + Files []db.FileMetadata + TotalCount int64 + Page int + Limit int + TotalPages int + Job *db.Job // Optional: if viewing files for a specific job + Filter FileMetadataFilter +} + +// FileMetadataDetailsData contains data for the file metadata details template +type FileMetadataDetailsData struct { + File db.FileMetadata +} + +// FileMetadataSearchData contains data for the file metadata search template +type FileMetadataSearchData struct { + Files []db.FileMetadata + TotalCount int64 + Page int + Limit int + TotalPages int + Filter FileMetadataFilter +} + +// getStatusBadgeClass returns the appropriate CSS class for a file status badge +func getStatusBadgeClass(status string) string { + switch status { + case "processed": + return "badge-success" + case "archived": + return "badge-info" + case "deleted": + return "badge-warning" + case "archived_and_deleted": + return "badge-warning" + case "error": + return "badge-danger" + default: + return "badge-secondary" + } +} + +// formatFileSize formats a file size in bytes to a human-readable string +func formatFileSize(size int64) string { + if size < 1024 { + return fmt.Sprintf("%d B", size) + } else if size < 1024*1024 { + return fmt.Sprintf("%.2f KB", float64(size)/1024) + } else if size < 1024*1024*1024 { + return fmt.Sprintf("%.2f MB", float64(size)/(1024*1024)) + } else { + return fmt.Sprintf("%.2f GB", float64(size)/(1024*1024*1024)) + } +} + +// FileMetadataList renders a list of file metadata with pagination and filters +templ FileMetadataList(ctx context.Context, data FileMetadataListData) { + @LayoutWithContext("File Metadata", ctx) { +
+
+

+ if data.Job != nil { + Files for Job: { data.Job.Name } + } else { + File Metadata + } +

+
+ + Advanced Search + +
+
+ + +
+
+

Filter Files

+
+
+
+ if data.Job == nil { +
+ + +
+ } +
+ + +
+
+ + +
+
+ +
+
+
+
+ + +
+
+

Files ({ strconv.FormatInt(data.TotalCount, 10) })

+
+ Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) } +
+
+
+ if len(data.Files) > 0 { + + + + + + + + + if data.Job == nil { + + } + + + + + for _, file := range data.Files { + + + + + + + if data.Job == nil && file.Job.ID > 0 { + + } + + + } + +
IDFilenameSizeProcessedStatusJobActions
{ strconv.FormatUint(uint64(file.ID), 10) } + + { file.FileName } + + { formatFileSize(file.FileSize) }{ file.ProcessedTime.Format("2006-01-02 15:04:05") } + + { file.Status } + + + + { file.Job.Name } + + + + + + +
+ + + if data.TotalPages > 1 { +
+
+ if data.Page > 1 { + + + + } + + for i := 1; i <= data.TotalPages; i++ { + if i == data.Page { + { strconv.Itoa(i) } + } else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) { + + { strconv.Itoa(i) } + + } else if i == data.Page-3 || i == data.Page+3 { + ... + } + } + + if data.Page < data.TotalPages { + + + + } +
+
+ } + } else { +
+ +

No file metadata found matching your criteria.

+
+ } +
+
+
+ } +} + +// buildPaginationURL builds a URL for pagination links +func buildPaginationURL(data FileMetadataListData, page int) string { + baseURL := "/files" + if data.Job != nil { + baseURL = fmt.Sprintf("/files/job/%d", data.Job.ID) + } + + url := fmt.Sprintf("%s?page=%d&limit=%d", baseURL, page, data.Limit) + + if data.Filter.Status != "" { + url += "&status=" + data.Filter.Status + } + + if data.Filter.FileName != "" { + url += "&filename=" + data.Filter.FileName + } + + if data.Filter.JobID != "" && data.Job == nil { + url += "&job_id=" + data.Filter.JobID + } + + return url +} + +// FileMetadataDetails renders detailed information about a file +templ FileMetadataDetails(ctx context.Context, data FileMetadataDetailsData) { + @LayoutWithContext("File Details", ctx) { +
+
+ + Back to Files + +
+ +
+
+

+ File: { data.File.FileName } +

+
+
+
+
+

File Information

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ID{ strconv.FormatUint(uint64(data.File.ID), 10) }
Filename{ data.File.FileName }
Size{ formatFileSize(data.File.FileSize) }
Hash + if data.File.FileHash != "" { + { data.File.FileHash } + } else { + Not available + } +
Status + + { data.File.Status } + +
Original Path{ data.File.OriginalPath }
Destination Path{ data.File.DestinationPath }
+
+ +
+

Processing Information

+ + + + + + + + + + + + + + + + + + + if data.File.Status == "error" && data.File.ErrorMessage != "" { + + + + + } + + + + + + + + + +
Job + + { data.File.Job.Name } + +
Processed Time{ data.File.ProcessedTime.Format("2006-01-02 15:04:05") }
Creation Time{ data.File.CreationTime.Format("2006-01-02 15:04:05") }
Modification Time{ data.File.ModTime.Format("2006-01-02 15:04:05") }
Error{ data.File.ErrorMessage }
Record Created{ data.File.CreatedAt.Format("2006-01-02 15:04:05") }
Record Updated{ data.File.UpdatedAt.Format("2006-01-02 15:04:05") }
+
+
+ +
+ + Back to Files + + +
+
+
+
+ } +} + +// FileMetadataSearch renders an advanced search form for file metadata +templ FileMetadataSearch(ctx context.Context, data FileMetadataSearchData) { + @LayoutWithContext("Search Files", ctx) { +
+
+ + Back to Files + +
+ +
+
+

+ Advanced File Search +

+
+
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+
+
+ + + if data.TotalCount > 0 || (data.Filter.FileName != "" || data.Filter.JobID != "" || data.Filter.Status != "" || + data.Filter.Hash != "" || data.Filter.StartDate != "" || data.Filter.EndDate != "") { +
+
+

Search Results ({ strconv.FormatInt(data.TotalCount, 10) })

+
+ Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) } +
+
+
+ if len(data.Files) > 0 { + + + + + + + + + + + + + + for _, file := range data.Files { + + + + + + + + + + } + +
IDFilenameSizeProcessedStatusJobActions
{ strconv.FormatUint(uint64(file.ID), 10) } + + { file.FileName } + + { formatFileSize(file.FileSize) }{ file.ProcessedTime.Format("2006-01-02 15:04:05") } + + { file.Status } + + + + { file.Job.Name } + + + + + + +
+ + + if data.TotalPages > 1 { +
+
+ if data.Page > 1 { + + + + } + + for i := 1; i <= data.TotalPages; i++ { + if i == data.Page { + { strconv.Itoa(i) } + } else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) { + + { strconv.Itoa(i) } + + } else if i == data.Page-3 || i == data.Page+3 { + ... + } + } + + if data.Page < data.TotalPages { + + + + } +
+
+ } + } else { +
+ +

No files found matching your search criteria.

+
+ } +
+
+ } +
+ } +} + +// buildSearchPaginationURL builds a URL for search pagination links +func buildSearchPaginationURL(data FileMetadataSearchData, page int) string { + url := fmt.Sprintf("/files/search?page=%d&limit=%d", page, data.Limit) + + if data.Filter.Status != "" { + url += "&status=" + data.Filter.Status + } + + if data.Filter.FileName != "" { + url += "&filename=" + data.Filter.FileName + } + + if data.Filter.JobID != "" { + url += "&job_id=" + data.Filter.JobID + } + + if data.Filter.Hash != "" { + url += "&hash=" + data.Filter.Hash + } + + if data.Filter.StartDate != "" { + url += "&start_date=" + data.Filter.StartDate + } + + if data.Filter.EndDate != "" { + url += "&end_date=" + data.Filter.EndDate + } + + return url +} \ No newline at end of file diff --git a/components/layout.templ b/components/layout.templ index 395a3f1..1a50f86 100644 --- a/components/layout.templ +++ b/components/layout.templ @@ -300,6 +300,9 @@ templ LayoutWithContext(title string, ctx context.Context) { History + + Files + if isAdmin(ctx) { Users @@ -393,6 +396,9 @@ templ LayoutWithContext(title string, ctx context.Context) { History + + Files + if isAdmin(ctx) { Users @@ -446,6 +452,10 @@ templ LayoutWithContext(title string, ctx context.Context) { History + + + Files +