mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-09-16 03:20:55 +02:00
feat: Add file metadata tracking and search functionality
- Implement FileMetadata model to track processed files - Create file metadata handlers for listing, searching, and viewing files - Add file metadata routes and UI components - Support advanced file search with multiple filters - Enhance job execution to capture file metadata during transfers - Implement file hash and duplicate detection logic
This commit is contained in:
@@ -0,0 +1,587 @@
|
|||||||
|
package components
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"fmt"
|
||||||
|
"github.com/starfleetcptn/gomft/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FileMetadataFilter represents filter parameters for file metadata queries
|
||||||
|
type FileMetadataFilter struct {
|
||||||
|
Status string
|
||||||
|
JobID string
|
||||||
|
FileName string
|
||||||
|
Hash string
|
||||||
|
StartDate string
|
||||||
|
EndDate string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataListData contains data for the file metadata list template
|
||||||
|
type FileMetadataListData struct {
|
||||||
|
Files []db.FileMetadata
|
||||||
|
TotalCount int64
|
||||||
|
Page int
|
||||||
|
Limit int
|
||||||
|
TotalPages int
|
||||||
|
Job *db.Job // Optional: if viewing files for a specific job
|
||||||
|
Filter FileMetadataFilter
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataDetailsData contains data for the file metadata details template
|
||||||
|
type FileMetadataDetailsData struct {
|
||||||
|
File db.FileMetadata
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataSearchData contains data for the file metadata search template
|
||||||
|
type FileMetadataSearchData struct {
|
||||||
|
Files []db.FileMetadata
|
||||||
|
TotalCount int64
|
||||||
|
Page int
|
||||||
|
Limit int
|
||||||
|
TotalPages int
|
||||||
|
Filter FileMetadataFilter
|
||||||
|
}
|
||||||
|
|
||||||
|
// getStatusBadgeClass returns the appropriate CSS class for a file status badge
|
||||||
|
func getStatusBadgeClass(status string) string {
|
||||||
|
switch status {
|
||||||
|
case "processed":
|
||||||
|
return "badge-success"
|
||||||
|
case "archived":
|
||||||
|
return "badge-info"
|
||||||
|
case "deleted":
|
||||||
|
return "badge-warning"
|
||||||
|
case "archived_and_deleted":
|
||||||
|
return "badge-warning"
|
||||||
|
case "error":
|
||||||
|
return "badge-danger"
|
||||||
|
default:
|
||||||
|
return "badge-secondary"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// formatFileSize formats a file size in bytes to a human-readable string
|
||||||
|
func formatFileSize(size int64) string {
|
||||||
|
if size < 1024 {
|
||||||
|
return fmt.Sprintf("%d B", size)
|
||||||
|
} else if size < 1024*1024 {
|
||||||
|
return fmt.Sprintf("%.2f KB", float64(size)/1024)
|
||||||
|
} else if size < 1024*1024*1024 {
|
||||||
|
return fmt.Sprintf("%.2f MB", float64(size)/(1024*1024))
|
||||||
|
} else {
|
||||||
|
return fmt.Sprintf("%.2f GB", float64(size)/(1024*1024*1024))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataList renders a list of file metadata with pagination and filters
|
||||||
|
templ FileMetadataList(ctx context.Context, data FileMetadataListData) {
|
||||||
|
@LayoutWithContext("File Metadata", ctx) {
|
||||||
|
<div class="container mx-auto px-4 py-8 animate-fadeIn">
|
||||||
|
<div class="flex justify-between items-center mb-6">
|
||||||
|
<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
|
||||||
|
if data.Job != nil {
|
||||||
|
Files for Job: { data.Job.Name }
|
||||||
|
} else {
|
||||||
|
File Metadata
|
||||||
|
}
|
||||||
|
</h1>
|
||||||
|
<div class="flex space-x-2">
|
||||||
|
<a href="/files/search" class="btn-primary">
|
||||||
|
<i class="fas fa-search mr-2"></i> Advanced Search
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Filter Form -->
|
||||||
|
<div class="card mb-6">
|
||||||
|
<div class="card-header">
|
||||||
|
<h2 class="text-lg font-semibold">Filter Files</h2>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<form method="GET" class="grid grid-cols-1 md:grid-cols-3 gap-4">
|
||||||
|
if data.Job == nil {
|
||||||
|
<div>
|
||||||
|
<label for="job_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job</label>
|
||||||
|
<input type="text" id="job_id" name="job_id" value={ data.Filter.JobID } placeholder="Job ID"
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
<div>
|
||||||
|
<label for="status" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Status</label>
|
||||||
|
<select id="status" name="status" class="form-input w-full">
|
||||||
|
<option value="">All Statuses</option>
|
||||||
|
<option value="processed" selected?={ data.Filter.Status == "processed" }>Processed</option>
|
||||||
|
<option value="archived" selected?={ data.Filter.Status == "archived" }>Archived</option>
|
||||||
|
<option value="deleted" selected?={ data.Filter.Status == "deleted" }>Deleted</option>
|
||||||
|
<option value="archived_and_deleted" selected?={ data.Filter.Status == "archived_and_deleted" }>Archived & Deleted</option>
|
||||||
|
<option value="error" selected?={ data.Filter.Status == "error" }>Error</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="filename" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Filename</label>
|
||||||
|
<input type="text" id="filename" name="filename" value={ data.Filter.FileName } placeholder="Filename or partial match"
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div class="md:col-span-3 flex justify-end">
|
||||||
|
<button type="submit" class="btn-primary">
|
||||||
|
<i class="fas fa-filter mr-2"></i> Apply Filters
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Results -->
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header flex justify-between items-center">
|
||||||
|
<h2 class="text-lg font-semibold">Files ({ strconv.FormatInt(data.TotalCount, 10) })</h2>
|
||||||
|
<div class="text-sm text-secondary-600 dark:text-secondary-400">
|
||||||
|
Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) }
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
if len(data.Files) > 0 {
|
||||||
|
<table class="table w-full">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Filename</th>
|
||||||
|
<th>Size</th>
|
||||||
|
<th>Processed</th>
|
||||||
|
<th>Status</th>
|
||||||
|
if data.Job == nil {
|
||||||
|
<th>Job</th>
|
||||||
|
}
|
||||||
|
<th>Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
for _, file := range data.Files {
|
||||||
|
<tr class="hover:bg-secondary-50 dark:hover:bg-secondary-800 transition-colors duration-150">
|
||||||
|
<td>{ strconv.FormatUint(uint64(file.ID), 10) }</td>
|
||||||
|
<td class="text-primary-600 dark:text-primary-400">
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="hover:underline">
|
||||||
|
{ file.FileName }
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>{ formatFileSize(file.FileSize) }</td>
|
||||||
|
<td>{ file.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
<td>
|
||||||
|
<span class={ "badge", getStatusBadgeClass(file.Status) }>
|
||||||
|
{ file.Status }
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
if data.Job == nil && file.Job.ID > 0 {
|
||||||
|
<td>
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", file.Job.ID)) } class="hover:underline text-primary-600 dark:text-primary-400">
|
||||||
|
{ file.Job.Name }
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
}
|
||||||
|
<td class="flex space-x-2">
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="text-primary-600 dark:text-primary-400 hover:text-primary-800 dark:hover:text-primary-300">
|
||||||
|
<i class="fas fa-eye"></i>
|
||||||
|
</a>
|
||||||
|
<button
|
||||||
|
class="text-red-600 dark:text-red-400 hover:text-red-800 dark:hover:text-red-300"
|
||||||
|
hx-delete={ fmt.Sprintf("/files/%d", file.ID) }
|
||||||
|
hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
|
||||||
|
hx-target="body">
|
||||||
|
<i class="fas fa-trash"></i>
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<!-- Pagination -->
|
||||||
|
if data.TotalPages > 1 {
|
||||||
|
<div class="flex justify-center items-center py-4 bg-secondary-50 dark:bg-secondary-800">
|
||||||
|
<div class="flex space-x-1">
|
||||||
|
if data.Page > 1 {
|
||||||
|
<a href={ templ.SafeURL(buildPaginationURL(data, data.Page - 1)) } class="btn-secondary">
|
||||||
|
<i class="fas fa-chevron-left"></i>
|
||||||
|
</a>
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 1; i <= data.TotalPages; i++ {
|
||||||
|
if i == data.Page {
|
||||||
|
<span class="px-3 py-2 bg-primary-600 text-white rounded">{ strconv.Itoa(i) }</span>
|
||||||
|
} else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) {
|
||||||
|
<a href={ templ.SafeURL(buildPaginationURL(data, i)) } class="px-3 py-2 bg-secondary-200 text-secondary-800 hover:bg-secondary-300 rounded dark:bg-secondary-700 dark:text-secondary-200 dark:hover:bg-secondary-600">
|
||||||
|
{ strconv.Itoa(i) }
|
||||||
|
</a>
|
||||||
|
} else if i == data.Page-3 || i == data.Page+3 {
|
||||||
|
<span class="px-3 py-2">...</span>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Page < data.TotalPages {
|
||||||
|
<a href={ templ.SafeURL(buildPaginationURL(data, data.Page + 1)) } class="btn-secondary">
|
||||||
|
<i class="fas fa-chevron-right"></i>
|
||||||
|
</a>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
<div class="p-6 text-center text-secondary-600 dark:text-secondary-400">
|
||||||
|
<i class="fas fa-file-alt text-5xl mb-3"></i>
|
||||||
|
<p>No file metadata found matching your criteria.</p>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildPaginationURL builds a URL for pagination links
|
||||||
|
func buildPaginationURL(data FileMetadataListData, page int) string {
|
||||||
|
baseURL := "/files"
|
||||||
|
if data.Job != nil {
|
||||||
|
baseURL = fmt.Sprintf("/files/job/%d", data.Job.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
url := fmt.Sprintf("%s?page=%d&limit=%d", baseURL, page, data.Limit)
|
||||||
|
|
||||||
|
if data.Filter.Status != "" {
|
||||||
|
url += "&status=" + data.Filter.Status
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.FileName != "" {
|
||||||
|
url += "&filename=" + data.Filter.FileName
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.JobID != "" && data.Job == nil {
|
||||||
|
url += "&job_id=" + data.Filter.JobID
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataDetails renders detailed information about a file
|
||||||
|
templ FileMetadataDetails(ctx context.Context, data FileMetadataDetailsData) {
|
||||||
|
@LayoutWithContext("File Details", ctx) {
|
||||||
|
<div class="container mx-auto px-4 py-8 animate-fadeIn">
|
||||||
|
<div class="mb-6">
|
||||||
|
<a href="/files" class="text-primary-600 dark:text-primary-400 hover:underline">
|
||||||
|
<i class="fas fa-arrow-left mr-2"></i> Back to Files
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header">
|
||||||
|
<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
|
||||||
|
File: { data.File.FileName }
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||||
|
<div>
|
||||||
|
<h2 class="text-lg font-semibold mb-4 text-secondary-800 dark:text-secondary-200">File Information</h2>
|
||||||
|
<table class="w-full">
|
||||||
|
<tbody>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">ID</td>
|
||||||
|
<td class="py-2">{ strconv.FormatUint(uint64(data.File.ID), 10) }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Filename</td>
|
||||||
|
<td class="py-2">{ data.File.FileName }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Size</td>
|
||||||
|
<td class="py-2">{ formatFileSize(data.File.FileSize) }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Hash</td>
|
||||||
|
<td class="py-2 break-all">
|
||||||
|
if data.File.FileHash != "" {
|
||||||
|
{ data.File.FileHash }
|
||||||
|
} else {
|
||||||
|
<span class="text-secondary-500 dark:text-secondary-500 italic">Not available</span>
|
||||||
|
}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Status</td>
|
||||||
|
<td class="py-2">
|
||||||
|
<span class={ "badge", getStatusBadgeClass(data.File.Status) }>
|
||||||
|
{ data.File.Status }
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Original Path</td>
|
||||||
|
<td class="py-2 break-all">{ data.File.OriginalPath }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Destination Path</td>
|
||||||
|
<td class="py-2 break-all">{ data.File.DestinationPath }</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div>
|
||||||
|
<h2 class="text-lg font-semibold mb-4 text-secondary-800 dark:text-secondary-200">Processing Information</h2>
|
||||||
|
<table class="w-full">
|
||||||
|
<tbody>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Job</td>
|
||||||
|
<td class="py-2">
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", data.File.JobID)) } class="text-primary-600 dark:text-primary-400 hover:underline">
|
||||||
|
{ data.File.Job.Name }
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Processed Time</td>
|
||||||
|
<td class="py-2">{ data.File.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Creation Time</td>
|
||||||
|
<td class="py-2">{ data.File.CreationTime.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Modification Time</td>
|
||||||
|
<td class="py-2">{ data.File.ModTime.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
</tr>
|
||||||
|
if data.File.Status == "error" && data.File.ErrorMessage != "" {
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Error</td>
|
||||||
|
<td class="py-2 text-red-600 dark:text-red-400 break-all">{ data.File.ErrorMessage }</td>
|
||||||
|
</tr>
|
||||||
|
}
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Record Created</td>
|
||||||
|
<td class="py-2">{ data.File.CreatedAt.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
</tr>
|
||||||
|
<tr class="border-b border-secondary-200 dark:border-secondary-700">
|
||||||
|
<td class="py-2 font-medium text-secondary-700 dark:text-secondary-300">Record Updated</td>
|
||||||
|
<td class="py-2">{ data.File.UpdatedAt.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mt-8 flex justify-end space-x-4">
|
||||||
|
<a href="/files" class="btn-secondary">
|
||||||
|
<i class="fas fa-list mr-2"></i> Back to Files
|
||||||
|
</a>
|
||||||
|
<button
|
||||||
|
class="btn-danger"
|
||||||
|
hx-delete={ fmt.Sprintf("/files/%d", data.File.ID) }
|
||||||
|
hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
|
||||||
|
hx-target="body">
|
||||||
|
<i class="fas fa-trash mr-2"></i> Delete Record
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileMetadataSearch renders an advanced search form for file metadata
|
||||||
|
templ FileMetadataSearch(ctx context.Context, data FileMetadataSearchData) {
|
||||||
|
@LayoutWithContext("Search Files", ctx) {
|
||||||
|
<div class="container mx-auto px-4 py-8 animate-fadeIn">
|
||||||
|
<div class="mb-6">
|
||||||
|
<a href="/files" class="text-primary-600 dark:text-primary-400 hover:underline">
|
||||||
|
<i class="fas fa-arrow-left mr-2"></i> Back to Files
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card mb-6">
|
||||||
|
<div class="card-header">
|
||||||
|
<h1 class="text-2xl font-bold text-secondary-800 dark:text-secondary-200">
|
||||||
|
Advanced File Search
|
||||||
|
</h1>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<form method="GET" class="grid grid-cols-1 md:grid-cols-2 gap-6">
|
||||||
|
<div>
|
||||||
|
<label for="job_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job ID</label>
|
||||||
|
<input type="text" id="job_id" name="job_id" value={ data.Filter.JobID } placeholder="Job ID"
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="status" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Status</label>
|
||||||
|
<select id="status" name="status" class="form-input w-full">
|
||||||
|
<option value="">All Statuses</option>
|
||||||
|
<option value="processed" selected?={ data.Filter.Status == "processed" }>Processed</option>
|
||||||
|
<option value="archived" selected?={ data.Filter.Status == "archived" }>Archived</option>
|
||||||
|
<option value="deleted" selected?={ data.Filter.Status == "deleted" }>Deleted</option>
|
||||||
|
<option value="archived_and_deleted" selected?={ data.Filter.Status == "archived_and_deleted" }>Archived & Deleted</option>
|
||||||
|
<option value="error" selected?={ data.Filter.Status == "error" }>Error</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="filename" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Filename</label>
|
||||||
|
<input type="text" id="filename" name="filename" value={ data.Filter.FileName } placeholder="Filename or partial match"
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="hash" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">File Hash</label>
|
||||||
|
<input type="text" id="hash" name="hash" value={ data.Filter.Hash } placeholder="MD5 hash"
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="start_date" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Processed After</label>
|
||||||
|
<input type="date" id="start_date" name="start_date" value={ data.Filter.StartDate }
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label for="end_date" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Processed Before</label>
|
||||||
|
<input type="date" id="end_date" name="end_date" value={ data.Filter.EndDate }
|
||||||
|
class="form-input w-full" />
|
||||||
|
</div>
|
||||||
|
<div class="md:col-span-2 flex justify-end">
|
||||||
|
<button type="submit" class="btn-primary">
|
||||||
|
<i class="fas fa-search mr-2"></i> Search Files
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Results -->
|
||||||
|
if data.TotalCount > 0 || (data.Filter.FileName != "" || data.Filter.JobID != "" || data.Filter.Status != "" ||
|
||||||
|
data.Filter.Hash != "" || data.Filter.StartDate != "" || data.Filter.EndDate != "") {
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header flex justify-between items-center">
|
||||||
|
<h2 class="text-lg font-semibold">Search Results ({ strconv.FormatInt(data.TotalCount, 10) })</h2>
|
||||||
|
<div class="text-sm text-secondary-600 dark:text-secondary-400">
|
||||||
|
Page { strconv.Itoa(data.Page) } of { strconv.Itoa(data.TotalPages) }
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="overflow-x-auto">
|
||||||
|
if len(data.Files) > 0 {
|
||||||
|
<table class="table w-full">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>ID</th>
|
||||||
|
<th>Filename</th>
|
||||||
|
<th>Size</th>
|
||||||
|
<th>Processed</th>
|
||||||
|
<th>Status</th>
|
||||||
|
<th>Job</th>
|
||||||
|
<th>Actions</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
for _, file := range data.Files {
|
||||||
|
<tr class="hover:bg-secondary-50 dark:hover:bg-secondary-800 transition-colors duration-150">
|
||||||
|
<td>{ strconv.FormatUint(uint64(file.ID), 10) }</td>
|
||||||
|
<td class="text-primary-600 dark:text-primary-400">
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="hover:underline">
|
||||||
|
{ file.FileName }
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>{ formatFileSize(file.FileSize) }</td>
|
||||||
|
<td>{ file.ProcessedTime.Format("2006-01-02 15:04:05") }</td>
|
||||||
|
<td>
|
||||||
|
<span class={ "badge", getStatusBadgeClass(file.Status) }>
|
||||||
|
{ file.Status }
|
||||||
|
</span>
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/job/%d", file.Job.ID)) } class="hover:underline text-primary-600 dark:text-primary-400">
|
||||||
|
{ file.Job.Name }
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td class="flex space-x-2">
|
||||||
|
<a href={ templ.SafeURL(fmt.Sprintf("/files/%d", file.ID)) } class="text-primary-600 dark:text-primary-400 hover:text-primary-800 dark:hover:text-primary-300">
|
||||||
|
<i class="fas fa-eye"></i>
|
||||||
|
</a>
|
||||||
|
<button
|
||||||
|
class="text-red-600 dark:text-red-400 hover:text-red-800 dark:hover:text-red-300"
|
||||||
|
hx-delete={ fmt.Sprintf("/files/%d", file.ID) }
|
||||||
|
hx-confirm="Are you sure you want to delete this file metadata record? This cannot be undone."
|
||||||
|
hx-target="body">
|
||||||
|
<i class="fas fa-trash"></i>
|
||||||
|
</button>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<!-- Pagination -->
|
||||||
|
if data.TotalPages > 1 {
|
||||||
|
<div class="flex justify-center items-center py-4 bg-secondary-50 dark:bg-secondary-800">
|
||||||
|
<div class="flex space-x-1">
|
||||||
|
if data.Page > 1 {
|
||||||
|
<a href={ templ.SafeURL(buildSearchPaginationURL(data, data.Page - 1)) } class="btn-secondary">
|
||||||
|
<i class="fas fa-chevron-left"></i>
|
||||||
|
</a>
|
||||||
|
}
|
||||||
|
|
||||||
|
for i := 1; i <= data.TotalPages; i++ {
|
||||||
|
if i == data.Page {
|
||||||
|
<span class="px-3 py-2 bg-primary-600 text-white rounded">{ strconv.Itoa(i) }</span>
|
||||||
|
} else if i == 1 || i == data.TotalPages || (i >= data.Page-2 && i <= data.Page+2) {
|
||||||
|
<a href={ templ.SafeURL(buildSearchPaginationURL(data, i)) } class="px-3 py-2 bg-secondary-200 text-secondary-800 hover:bg-secondary-300 rounded dark:bg-secondary-700 dark:text-secondary-200 dark:hover:bg-secondary-600">
|
||||||
|
{ strconv.Itoa(i) }
|
||||||
|
</a>
|
||||||
|
} else if i == data.Page-3 || i == data.Page+3 {
|
||||||
|
<span class="px-3 py-2">...</span>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Page < data.TotalPages {
|
||||||
|
<a href={ templ.SafeURL(buildSearchPaginationURL(data, data.Page + 1)) } class="btn-secondary">
|
||||||
|
<i class="fas fa-chevron-right"></i>
|
||||||
|
</a>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
<div class="p-6 text-center text-secondary-600 dark:text-secondary-400">
|
||||||
|
<i class="fas fa-search text-5xl mb-3"></i>
|
||||||
|
<p>No files found matching your search criteria.</p>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildSearchPaginationURL builds a URL for search pagination links
|
||||||
|
func buildSearchPaginationURL(data FileMetadataSearchData, page int) string {
|
||||||
|
url := fmt.Sprintf("/files/search?page=%d&limit=%d", page, data.Limit)
|
||||||
|
|
||||||
|
if data.Filter.Status != "" {
|
||||||
|
url += "&status=" + data.Filter.Status
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.FileName != "" {
|
||||||
|
url += "&filename=" + data.Filter.FileName
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.JobID != "" {
|
||||||
|
url += "&job_id=" + data.Filter.JobID
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.Hash != "" {
|
||||||
|
url += "&hash=" + data.Filter.Hash
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.StartDate != "" {
|
||||||
|
url += "&start_date=" + data.Filter.StartDate
|
||||||
|
}
|
||||||
|
|
||||||
|
if data.Filter.EndDate != "" {
|
||||||
|
url += "&end_date=" + data.Filter.EndDate
|
||||||
|
}
|
||||||
|
|
||||||
|
return url
|
||||||
|
}
|
||||||
@@ -300,6 +300,9 @@ templ LayoutWithContext(title string, ctx context.Context) {
|
|||||||
<a href="/history" class="nav-link">
|
<a href="/history" class="nav-link">
|
||||||
<i class="fas fa-history mr-1"></i> History
|
<i class="fas fa-history mr-1"></i> History
|
||||||
</a>
|
</a>
|
||||||
|
<a href="/files" class="nav-link">
|
||||||
|
<i class="fas fa-file-alt mr-1"></i> Files
|
||||||
|
</a>
|
||||||
if isAdmin(ctx) {
|
if isAdmin(ctx) {
|
||||||
<a href="/admin/users" class="nav-link">
|
<a href="/admin/users" class="nav-link">
|
||||||
<i class="fas fa-users mr-1"></i> Users
|
<i class="fas fa-users mr-1"></i> Users
|
||||||
@@ -393,6 +396,9 @@ templ LayoutWithContext(title string, ctx context.Context) {
|
|||||||
<a href="/history" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
|
<a href="/history" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
|
||||||
<i class="fas fa-history mr-2"></i> History
|
<i class="fas fa-history mr-2"></i> History
|
||||||
</a>
|
</a>
|
||||||
|
<a href="/files" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
|
||||||
|
<i class="fas fa-file-alt mr-2"></i> Files
|
||||||
|
</a>
|
||||||
if isAdmin(ctx) {
|
if isAdmin(ctx) {
|
||||||
<a href="/admin/users" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
|
<a href="/admin/users" class="block px-3 py-2 rounded-md text-base font-medium text-secondary-700 dark:text-secondary-300 hover:bg-primary-50 dark:hover:bg-secondary-700 hover:text-primary-600 dark:hover:text-primary-400">
|
||||||
<i class="fas fa-users mr-2"></i> Users
|
<i class="fas fa-users mr-2"></i> Users
|
||||||
@@ -446,6 +452,10 @@ templ LayoutWithContext(title string, ctx context.Context) {
|
|||||||
<i class="fas fa-history text-lg"></i>
|
<i class="fas fa-history text-lg"></i>
|
||||||
<span class="text-xs mt-1">History</span>
|
<span class="text-xs mt-1">History</span>
|
||||||
</a>
|
</a>
|
||||||
|
<a href="/files" class="flex flex-col items-center justify-center text-secondary-500 dark:text-secondary-400 hover:text-primary-600 dark:hover:text-primary-400">
|
||||||
|
<i class="fas fa-file-alt text-lg"></i>
|
||||||
|
<span class="text-xs mt-1">Files</span>
|
||||||
|
</a>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
class="flex flex-col items-center justify-center text-secondary-500 dark:text-secondary-400 hover:text-primary-600 dark:hover:text-primary-400"
|
class="flex flex-col items-center justify-center text-secondary-500 dark:text-secondary-400 hover:text-primary-600 dark:hover:text-primary-400"
|
||||||
|
|||||||
@@ -65,45 +65,63 @@ templ FilePatternFields() {
|
|||||||
}
|
}
|
||||||
|
|
||||||
templ ArchiveOptions() {
|
templ ArchiveOptions() {
|
||||||
<div class="space-y-6">
|
<div class="border border-gray-200 rounded-lg p-6 bg-gray-50 shadow-sm">
|
||||||
<div class="flex items-start mb-4">
|
<h3 class="text-lg font-medium text-gray-900 mb-4">Archive & Delete Options</h3>
|
||||||
<div class="flex items-center h-5">
|
|
||||||
<input id="archive_enabled" name="archive_enabled" type="checkbox" x-model="archiveEnabled"
|
|
||||||
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
|
||||||
</div>
|
|
||||||
<div class="ml-3 text-sm">
|
|
||||||
<label for="archive_enabled" class="font-medium text-secondary-700 dark:text-secondary-300">Enable
|
|
||||||
Archiving</label>
|
|
||||||
<p class="text-secondary-500 dark:text-secondary-400">Archive files after successful transfer</p>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div x-show="archiveEnabled" class="sm:col-span-4">
|
<div class="mb-4">
|
||||||
<label for="archive_path"
|
<label for="archive_enabled" class="flex items-center cursor-pointer">
|
||||||
class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Archive Path</label>
|
<div class="relative">
|
||||||
<div class="relative">
|
<input id="archive_enabled" name="archive_enabled" type="checkbox" x-model="archiveEnabled"
|
||||||
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
|
class="sr-only"
|
||||||
<i class="fas fa-archive text-secondary-400 dark:text-secondary-600"></i>
|
:value="archiveEnabled ? 'true' : 'false'"
|
||||||
|
/>
|
||||||
|
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
|
||||||
|
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
|
||||||
|
:class="archiveEnabled ? 'transform translate-x-6 bg-primary-500' : ''"></div>
|
||||||
|
</div>
|
||||||
|
<div class="ml-3 text-gray-700 font-medium">
|
||||||
|
Enable archiving
|
||||||
|
</div>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="mb-4" x-show="archiveEnabled">
|
||||||
|
<label for="archive_path" class="block text-sm font-medium text-gray-700 mb-1">Archive Path</label>
|
||||||
|
<div class="relative">
|
||||||
|
<span class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
|
||||||
|
<i class="fas fa-folder text-gray-400"></i>
|
||||||
|
</span>
|
||||||
|
<input id="archive_path" name="archive_path" type="text"
|
||||||
|
class="pl-10 pr-10 py-2 border border-gray-300 rounded-md focus:ring-primary-500 focus:border-primary-500 block w-full shadow-sm"
|
||||||
|
placeholder="Path to archive files"
|
||||||
|
x-model="archivePath"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
<input type="text" name="archive_path" id="archive_path" x-model="archivePath"
|
<p class="mt-1 text-xs text-gray-500">
|
||||||
x-bind:required="archiveEnabled"
|
Files will be moved here after successful transfer
|
||||||
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
|
</p>
|
||||||
placeholder="/path/to/archive" />
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
|
|
||||||
<div class="flex items-start mb-4">
|
<div class="mb-4">
|
||||||
<div class="flex items-center h-5">
|
<label for="delete_after_transfer" class="flex items-center cursor-pointer">
|
||||||
<input id="delete_after_transfer" name="delete_after_transfer" type="checkbox" x-model="deleteAfterTransfer"
|
<div class="relative">
|
||||||
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
<input id="delete_after_transfer" name="delete_after_transfer" type="checkbox" x-model="deleteAfterTransfer"
|
||||||
</div>
|
class="sr-only"
|
||||||
<div class="ml-3 text-sm">
|
:value="deleteAfterTransfer ? 'true' : 'false'"
|
||||||
<label for="delete_after_transfer" class="font-medium text-secondary-700 dark:text-secondary-300">Delete
|
/>
|
||||||
After Transfer</label>
|
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
|
||||||
<p class="text-secondary-500 dark:text-secondary-400">Delete source files after successful transfer</p>
|
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
|
||||||
|
:class="deleteAfterTransfer ? 'transform translate-x-6 bg-red-500' : ''"></div>
|
||||||
|
</div>
|
||||||
|
<div class="ml-3 text-gray-700 font-medium">
|
||||||
|
Delete source files after transfer
|
||||||
|
</div>
|
||||||
|
</label>
|
||||||
|
<p class="mt-1 ml-14 text-xs text-red-500" x-show="deleteAfterTransfer">
|
||||||
|
<i class="fas fa-exclamation-triangle mr-1"></i> Warning: This will permanently delete the original files
|
||||||
|
</p>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
|
||||||
}
|
}
|
||||||
|
|
||||||
templ RcloneFlags() {
|
templ RcloneFlags() {
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ templ FTPDestinationForm() {
|
|||||||
name="dest_passive_mode"
|
name="dest_passive_mode"
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
x-model="destPassiveMode"
|
x-model="destPassiveMode"
|
||||||
|
:value="destPassiveMode ? 'true' : 'false'"
|
||||||
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
||||||
</div>
|
</div>
|
||||||
<div class="ml-3 text-sm">
|
<div class="ml-3 text-sm">
|
||||||
|
|||||||
@@ -94,6 +94,7 @@ templ FTPSourceForm() {
|
|||||||
name="source_passive_mode"
|
name="source_passive_mode"
|
||||||
type="checkbox"
|
type="checkbox"
|
||||||
x-model="sourcePassiveMode"
|
x-model="sourcePassiveMode"
|
||||||
|
:value="sourcePassiveMode ? 'true' : 'false'"
|
||||||
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
class="focus:ring-primary-500 h-4 w-4 text-primary-600 border-secondary-300 dark:border-secondary-700 rounded">
|
||||||
</div>
|
</div>
|
||||||
<div class="ml-3 text-sm">
|
<div class="ml-3 text-sm">
|
||||||
|
|||||||
+123
-53
@@ -7,23 +7,23 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/starfleetcptn/gomft/internal/auth"
|
|
||||||
"github.com/glebarez/sqlite"
|
"github.com/glebarez/sqlite"
|
||||||
|
"github.com/starfleetcptn/gomft/internal/auth"
|
||||||
"gorm.io/gorm"
|
"gorm.io/gorm"
|
||||||
)
|
)
|
||||||
|
|
||||||
type User struct {
|
type User struct {
|
||||||
ID uint `gorm:"primarykey"`
|
ID uint `gorm:"primarykey"`
|
||||||
Email string `gorm:"unique;not null"`
|
Email string `gorm:"unique;not null"`
|
||||||
PasswordHash string `gorm:"not null"`
|
PasswordHash string `gorm:"not null"`
|
||||||
IsAdmin bool `gorm:"default:false"`
|
IsAdmin bool `gorm:"default:false"`
|
||||||
LastPasswordChange time.Time
|
LastPasswordChange time.Time
|
||||||
FailedLoginAttempts int `gorm:"default:0"`
|
FailedLoginAttempts int `gorm:"default:0"`
|
||||||
AccountLocked bool `gorm:"default:false"`
|
AccountLocked bool `gorm:"default:false"`
|
||||||
LockoutUntil *time.Time
|
LockoutUntil *time.Time
|
||||||
Theme string `gorm:"default:'light'"`
|
Theme string `gorm:"default:'light'"`
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
UpdatedAt time.Time
|
UpdatedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
type PasswordHistory struct {
|
type PasswordHistory struct {
|
||||||
@@ -46,31 +46,31 @@ type PasswordResetToken struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type TransferConfig struct {
|
type TransferConfig struct {
|
||||||
ID uint `gorm:"primarykey"`
|
ID uint `gorm:"primarykey"`
|
||||||
Name string `gorm:"not null" form:"name"`
|
Name string `gorm:"not null" form:"name"`
|
||||||
SourceType string `gorm:"not null" form:"source_type"`
|
SourceType string `gorm:"not null" form:"source_type"`
|
||||||
SourcePath string `gorm:"not null" form:"source_path"`
|
SourcePath string `gorm:"not null" form:"source_path"`
|
||||||
SourceHost string `form:"source_host"`
|
SourceHost string `form:"source_host"`
|
||||||
SourcePort int `gorm:"default:22" form:"source_port"`
|
SourcePort int `gorm:"default:22" form:"source_port"`
|
||||||
SourceUser string `form:"source_user"`
|
SourceUser string `form:"source_user"`
|
||||||
SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
|
SourcePassword string `form:"source_password" gorm:"-"` // Not stored in DB, only used for form
|
||||||
SourceKeyFile string `form:"source_key_file"`
|
SourceKeyFile string `form:"source_key_file"`
|
||||||
// S3 source fields
|
// S3 source fields
|
||||||
SourceBucket string `form:"source_bucket"`
|
SourceBucket string `form:"source_bucket"`
|
||||||
SourceRegion string `form:"source_region"`
|
SourceRegion string `form:"source_region"`
|
||||||
SourceAccessKey string `form:"source_access_key"`
|
SourceAccessKey string `form:"source_access_key"`
|
||||||
SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
|
SourceSecretKey string `form:"source_secret_key" gorm:"-"` // Not stored in DB, only used for form
|
||||||
SourceEndpoint string `form:"source_endpoint"`
|
SourceEndpoint string `form:"source_endpoint"`
|
||||||
// SMB source fields
|
// SMB source fields
|
||||||
SourceShare string `form:"source_share"`
|
SourceShare string `form:"source_share"`
|
||||||
SourceDomain string `form:"source_domain"`
|
SourceDomain string `form:"source_domain"`
|
||||||
// FTP source fields
|
// FTP source fields
|
||||||
SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
|
SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"`
|
||||||
// OneDrive and Google Drive source fields
|
// OneDrive and Google Drive source fields
|
||||||
SourceClientID string `form:"source_client_id"`
|
SourceClientID string `form:"source_client_id"`
|
||||||
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
||||||
SourceDriveID string `form:"source_drive_id"` // For OneDrive
|
SourceDriveID string `form:"source_drive_id"` // For OneDrive
|
||||||
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
|
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
|
||||||
// General fields
|
// General fields
|
||||||
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
|
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
|
||||||
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
|
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
|
||||||
@@ -82,30 +82,30 @@ type TransferConfig struct {
|
|||||||
DestPassword string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
|
DestPassword string `form:"dest_password" gorm:"-"` // Not stored in DB, only used for form
|
||||||
DestKeyFile string `form:"dest_key_file"`
|
DestKeyFile string `form:"dest_key_file"`
|
||||||
// S3 destination fields
|
// S3 destination fields
|
||||||
DestBucket string `form:"dest_bucket"`
|
DestBucket string `form:"dest_bucket"`
|
||||||
DestRegion string `form:"dest_region"`
|
DestRegion string `form:"dest_region"`
|
||||||
DestAccessKey string `form:"dest_access_key"`
|
DestAccessKey string `form:"dest_access_key"`
|
||||||
DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
|
DestSecretKey string `form:"dest_secret_key" gorm:"-"` // Not stored in DB, only used for form
|
||||||
DestEndpoint string `form:"dest_endpoint"`
|
DestEndpoint string `form:"dest_endpoint"`
|
||||||
// SMB destination fields
|
// SMB destination fields
|
||||||
DestShare string `form:"dest_share"`
|
DestShare string `form:"dest_share"`
|
||||||
DestDomain string `form:"dest_domain"`
|
DestDomain string `form:"dest_domain"`
|
||||||
// FTP destination fields
|
// FTP destination fields
|
||||||
DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
|
DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"`
|
||||||
// OneDrive and Google Drive destination fields
|
// OneDrive and Google Drive destination fields
|
||||||
DestClientID string `form:"dest_client_id"`
|
DestClientID string `form:"dest_client_id"`
|
||||||
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
||||||
DestDriveID string `form:"dest_drive_id"` // For OneDrive
|
DestDriveID string `form:"dest_drive_id"` // For OneDrive
|
||||||
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
|
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
|
||||||
// General fields
|
// General fields
|
||||||
ArchivePath string `form:"archive_path"`
|
ArchivePath string `form:"archive_path"`
|
||||||
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
|
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
|
||||||
RcloneFlags string `form:"rclone_flags"`
|
RcloneFlags string `form:"rclone_flags"`
|
||||||
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
|
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
|
||||||
CreatedBy uint
|
CreatedBy uint
|
||||||
User User `gorm:"foreignkey:CreatedBy"`
|
User User `gorm:"foreignkey:CreatedBy"`
|
||||||
CreatedAt time.Time
|
CreatedAt time.Time
|
||||||
UpdatedAt time.Time
|
UpdatedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
@@ -135,6 +135,25 @@ type JobHistory struct {
|
|||||||
ErrorMessage string
|
ErrorMessage string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FileMetadata stores information about processed files
|
||||||
|
type FileMetadata struct {
|
||||||
|
ID uint `gorm:"primarykey"`
|
||||||
|
JobID uint `gorm:"not null;index"`
|
||||||
|
Job Job `gorm:"foreignkey:JobID"`
|
||||||
|
FileName string `gorm:"not null"`
|
||||||
|
OriginalPath string `gorm:"not null"`
|
||||||
|
FileSize int64 `gorm:"not null"`
|
||||||
|
FileHash string `gorm:"index"` // MD5 or other hash for file identity
|
||||||
|
CreationTime time.Time
|
||||||
|
ModTime time.Time
|
||||||
|
ProcessedTime time.Time `gorm:"not null"`
|
||||||
|
DestinationPath string `gorm:"not null"`
|
||||||
|
Status string `gorm:"not null"` // processed, archived, deleted, etc.
|
||||||
|
ErrorMessage string
|
||||||
|
CreatedAt time.Time
|
||||||
|
UpdatedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
type DB struct {
|
type DB struct {
|
||||||
*gorm.DB
|
*gorm.DB
|
||||||
}
|
}
|
||||||
@@ -153,7 +172,7 @@ func Initialize(dbPath string) (*DB, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Auto migrate the schema
|
// Auto migrate the schema
|
||||||
err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{})
|
err = db.AutoMigrate(&User{}, &auth.PasswordHistory{}, &PasswordResetToken{}, &TransferConfig{}, &Job{}, &JobHistory{}, &FileMetadata{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to migrate database: %v", err)
|
return nil, fmt.Errorf("failed to migrate database: %v", err)
|
||||||
}
|
}
|
||||||
@@ -302,12 +321,63 @@ func (db *DB) UpdateJobHistory(history *JobHistory) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
|
func (db *DB) GetJobHistory(jobID uint) ([]JobHistory, error) {
|
||||||
var history []JobHistory
|
var histories []JobHistory
|
||||||
err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&history).Error
|
err := db.Where("job_id = ?", jobID).Order("start_time desc").Find(&histories).Error
|
||||||
return history, err
|
return histories, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// CreateFileMetadata creates a new file metadata record
|
||||||
|
func (db *DB) CreateFileMetadata(metadata *FileMetadata) error {
|
||||||
|
return db.Create(metadata).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadata retrieves file metadata by ID
|
||||||
|
func (db *DB) GetFileMetadata(id uint) (*FileMetadata, error) {
|
||||||
|
var metadata FileMetadata
|
||||||
|
err := db.First(&metadata, id).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadataByJobAndName retrieves file metadata by job ID and filename
|
||||||
|
func (db *DB) GetFileMetadataByJobAndName(jobID uint, fileName string) (*FileMetadata, error) {
|
||||||
|
var metadata FileMetadata
|
||||||
|
err := db.Where("job_id = ? AND file_name = ?", jobID, fileName).First(&metadata).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadataByHash retrieves file metadata by file hash
|
||||||
|
func (db *DB) GetFileMetadataByHash(fileHash string) (*FileMetadata, error) {
|
||||||
|
var metadata FileMetadata
|
||||||
|
err := db.Where("file_hash = ?", fileHash).First(&metadata).Error
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// UpdateFileMetadata updates an existing file metadata record
|
||||||
|
func (db *DB) UpdateFileMetadata(metadata *FileMetadata) error {
|
||||||
|
return db.Save(metadata).Error
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadataForJob retrieves all file metadata for a job
|
||||||
|
func (db *DB) GetFileMetadataForJob(jobID uint) ([]FileMetadata, error) {
|
||||||
|
var metadata []FileMetadata
|
||||||
|
err := db.Where("job_id = ?", jobID).Find(&metadata).Error
|
||||||
|
return metadata, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteFileMetadata deletes file metadata by ID
|
||||||
|
func (db *DB) DeleteFileMetadata(id uint) error {
|
||||||
|
return db.Delete(&FileMetadata{}, id).Error
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper functions
|
|
||||||
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
|
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
|
||||||
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
|
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,17 @@
|
|||||||
package scheduler
|
package scheduler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
"os"
|
"os"
|
||||||
"os/exec"
|
"os/exec"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -200,7 +205,16 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
|
|
||||||
totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
|
totalObjects := strings.TrimSpace(strings.Split(outputStr, "\n")[0])
|
||||||
totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
|
totalObjects = strings.TrimSpace(strings.Split(totalObjects, ":")[1])
|
||||||
// totalSize := strings.TrimSpace(strings.Split(outputStr, ":")[2])
|
totalSize := strings.TrimSpace(strings.Split(outputStr, "Total size:")[1])
|
||||||
|
if strings.Contains(totalSize, "(") {
|
||||||
|
totalSize = strings.TrimSpace(strings.Split(totalSize, "(")[1])
|
||||||
|
totalSize = strings.TrimSpace(strings.Split(totalSize, " ")[0])
|
||||||
|
} else {
|
||||||
|
totalSize = "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
bytesTransferred, _ := strconv.ParseInt(totalSize, 10, 64)
|
||||||
|
history.BytesTransferred = bytesTransferred
|
||||||
|
|
||||||
if totalObjects == "0" {
|
if totalObjects == "0" {
|
||||||
fmt.Printf("No files to transfer for job %d\n", jobID)
|
fmt.Printf("No files to transfer for job %d\n", jobID)
|
||||||
@@ -261,6 +275,100 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
|
|
||||||
fmt.Printf("Processing file: %s for job %d\n", file, jobID)
|
fmt.Printf("Processing file: %s for job %d\n", file, jobID)
|
||||||
|
|
||||||
|
// Get detailed file info if this is a local source
|
||||||
|
var fileSize int64
|
||||||
|
var createTime, modTime time.Time
|
||||||
|
var fileHash string
|
||||||
|
var fileMetadataErr error
|
||||||
|
|
||||||
|
// Get file metadata based on source type
|
||||||
|
if job.Config.SourceType == "local" {
|
||||||
|
// Construct the full local path
|
||||||
|
localFilePath := filepath.Join(job.Config.SourcePath, file)
|
||||||
|
|
||||||
|
// Get file info (size, creation time, modification time)
|
||||||
|
fileSize, createTime, modTime, fileMetadataErr = getFileInfo(localFilePath)
|
||||||
|
if fileMetadataErr != nil {
|
||||||
|
fmt.Printf("Warning: Could not get file info for %s: %v\n", file, fileMetadataErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Calculate file hash (MD5)
|
||||||
|
fileHash, fileMetadataErr = calculateFileHash(localFilePath)
|
||||||
|
if fileMetadataErr != nil {
|
||||||
|
fmt.Printf("Warning: Could not calculate hash for %s: %v\n", file, fileMetadataErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if this file has been processed before (by hash)
|
||||||
|
if fileHash != "" {
|
||||||
|
processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
|
||||||
|
if processed {
|
||||||
|
fmt.Printf("File %s has been processed before (hash: %s, previous file: %s)\n",
|
||||||
|
file, fileHash, prevMetadata.FileName)
|
||||||
|
|
||||||
|
// If configured to skip previously processed files,
|
||||||
|
// we could add that logic here
|
||||||
|
// For now, we'll just log it and continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check the processing history for this specific file name
|
||||||
|
prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
|
||||||
|
if histErr == nil {
|
||||||
|
fmt.Printf("File %s was previously processed on %s with status: %s\n",
|
||||||
|
file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
|
||||||
|
|
||||||
|
// If the file was previously processed successfully and the hash hasn't changed,
|
||||||
|
// we could skip processing
|
||||||
|
if prevMetadata.Status == "processed" ||
|
||||||
|
prevMetadata.Status == "archived" ||
|
||||||
|
prevMetadata.Status == "deleted" ||
|
||||||
|
prevMetadata.Status == "archived_and_deleted" {
|
||||||
|
if fileHash != "" && fileHash == prevMetadata.FileHash {
|
||||||
|
fmt.Printf("Skipping unchanged file %s (hash matches previous processing)\n", file)
|
||||||
|
// Skip this file and continue to the next one
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// For non-local sources, use rclone lsjson to get metadata
|
||||||
|
fileSize, createTime, modTime, fileHash, fileMetadataErr = s.getRemoteFileInfo(&job.Config, file)
|
||||||
|
if fileMetadataErr != nil {
|
||||||
|
fmt.Printf("Warning: Could not get remote file info for %s: %v\n", file, fileMetadataErr)
|
||||||
|
// We'll continue with placeholder values
|
||||||
|
fileSize = 0
|
||||||
|
createTime = time.Now()
|
||||||
|
modTime = time.Now()
|
||||||
|
fileHash = ""
|
||||||
|
} else {
|
||||||
|
// If we got a hash, check for previous processing
|
||||||
|
if fileHash != "" {
|
||||||
|
processed, prevMetadata, _ := s.hasFileBeenProcessed(jobID, fileHash)
|
||||||
|
if processed {
|
||||||
|
fmt.Printf("Remote file %s has been processed before (hash: %s, previous file: %s)\n",
|
||||||
|
file, fileHash, prevMetadata.FileName)
|
||||||
|
|
||||||
|
// Skip previously processed files with the same hash if they were processed successfully
|
||||||
|
if prevMetadata.Status == "processed" ||
|
||||||
|
prevMetadata.Status == "archived" ||
|
||||||
|
prevMetadata.Status == "deleted" ||
|
||||||
|
prevMetadata.Status == "archived_and_deleted" {
|
||||||
|
fmt.Printf("Skipping unchanged remote file %s (hash matches previous processing)\n", file)
|
||||||
|
// Skip this file and continue to the next one
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Also check by filename
|
||||||
|
prevMetadata, histErr := s.checkFileProcessingHistory(jobID, file)
|
||||||
|
if histErr == nil {
|
||||||
|
fmt.Printf("Remote file %s was previously processed on %s with status: %s\n",
|
||||||
|
file, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Prepare moveto command for transfer
|
// Prepare moveto command for transfer
|
||||||
transferArgs := []string{
|
transferArgs := []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
@@ -284,6 +392,8 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
|
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, file)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var destFile string = file
|
||||||
|
|
||||||
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
|
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
|
||||||
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
|
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, file)
|
||||||
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
|
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
|
||||||
@@ -296,9 +406,18 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
// Add output filename pattern if specified
|
// Add output filename pattern if specified
|
||||||
if job.Config.OutputPattern != "" {
|
if job.Config.OutputPattern != "" {
|
||||||
// Process the output pattern for this specific file
|
// Process the output pattern for this specific file
|
||||||
newFilename := ProcessOutputPattern(job.Config.OutputPattern, file)
|
destFile = ProcessOutputPattern(job.Config.OutputPattern, file)
|
||||||
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, newFilename)
|
|
||||||
fmt.Printf("Renaming file from %s to %s for job %d\n", file, newFilename, jobID)
|
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
|
||||||
|
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
|
||||||
|
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
|
||||||
|
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Printf("Renaming file from %s to %s for job %d\n", file, destFile, jobID)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add custom flags if specified
|
// Add custom flags if specified
|
||||||
@@ -325,14 +444,37 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
// Print the output
|
// Print the output
|
||||||
fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
|
fmt.Printf("Output for file %s: %s\n", file, string(fileOutput))
|
||||||
|
|
||||||
|
// Create file metadata record
|
||||||
|
fileStatus := "processed"
|
||||||
|
var fileErrorMsg string
|
||||||
|
var destPathForDB string
|
||||||
|
|
||||||
// Check if file was successfully transferred
|
// Check if file was successfully transferred
|
||||||
if fileErr != nil {
|
if fileErr != nil {
|
||||||
fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
|
fmt.Printf("Error transferring file %s for job %d: %v\n", file, jobID, fileErr)
|
||||||
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
|
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", file, fileErr))
|
||||||
|
fileStatus = "error"
|
||||||
|
fileErrorMsg = fileErr.Error()
|
||||||
} else {
|
} else {
|
||||||
filesTransferred++
|
filesTransferred++
|
||||||
fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
|
fmt.Printf("Successfully transferred file %s for job %d\n", file, jobID)
|
||||||
|
|
||||||
|
// Extract the actual destination path (without rclone remote prefix)
|
||||||
|
if job.Config.DestinationType == "local" {
|
||||||
|
destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
|
||||||
|
} else {
|
||||||
|
// For remote destinations, store the path format
|
||||||
|
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
|
||||||
|
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
|
||||||
|
destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
|
||||||
|
} else {
|
||||||
|
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// If archiving is enabled and transfer was successful, move files to archive
|
// If archiving is enabled and transfer was successful, move files to archive
|
||||||
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
|
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
|
||||||
fmt.Printf("Archiving file %s for job %d\n", file, jobID)
|
fmt.Printf("Archiving file %s for job %d\n", file, jobID)
|
||||||
@@ -372,14 +514,17 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
|
fmt.Printf("Warning: Error archiving file %s for job %d: %v\n", file, jobID, archiveErr)
|
||||||
transferErrors = append(transferErrors,
|
transferErrors = append(transferErrors,
|
||||||
fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
|
fmt.Sprintf("Archive error for file %s: %v", file, archiveErr))
|
||||||
|
} else {
|
||||||
|
fileStatus = "archived"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if job.Config.DeleteAfterTransfer {
|
if job.Config.DeleteAfterTransfer {
|
||||||
fmt.Printf("Deleting file %s for job %d\n", file, jobID)
|
fmt.Printf("Deleting file %s for job %d\n", file, jobID)
|
||||||
deleteArgs := []string{
|
deleteArgs := []string{
|
||||||
"--config", configPath,
|
"--config", configPath,
|
||||||
"deletefile",
|
"deletefile",
|
||||||
sourcePath, }
|
sourcePath}
|
||||||
deleteCmd := exec.Command(rclonePath, deleteArgs...)
|
deleteCmd := exec.Command(rclonePath, deleteArgs...)
|
||||||
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
|
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
|
||||||
fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
|
fmt.Printf("Output for file %s: %s\n", file, string(deleteOutput))
|
||||||
@@ -387,9 +532,36 @@ func (s *Scheduler) executeJob(jobID uint) {
|
|||||||
fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
|
fmt.Printf("Error deleting file %s for job %d: %v\n", file, jobID, deleteErr)
|
||||||
transferErrors = append(transferErrors,
|
transferErrors = append(transferErrors,
|
||||||
fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
|
fmt.Sprintf("Delete error for file %s: %v", file, deleteErr))
|
||||||
|
} else {
|
||||||
|
if fileStatus == "archived" {
|
||||||
|
fileStatus = "archived_and_deleted"
|
||||||
|
} else {
|
||||||
|
fileStatus = "deleted"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Create and save file metadata
|
||||||
|
metadata := &db.FileMetadata{
|
||||||
|
JobID: jobID,
|
||||||
|
FileName: file,
|
||||||
|
OriginalPath: job.Config.SourcePath,
|
||||||
|
FileSize: fileSize,
|
||||||
|
FileHash: fileHash,
|
||||||
|
CreationTime: createTime,
|
||||||
|
ModTime: modTime,
|
||||||
|
ProcessedTime: time.Now(),
|
||||||
|
DestinationPath: destPathForDB,
|
||||||
|
Status: fileStatus,
|
||||||
|
ErrorMessage: fileErrorMsg,
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.db.CreateFileMetadata(metadata); err != nil {
|
||||||
|
fmt.Printf("Error creating file metadata for %s: %v\n", file, err)
|
||||||
|
} else {
|
||||||
|
fmt.Printf("Created file metadata record for %s (ID: %d)\n", file, metadata.ID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Update job history with transfer results
|
// Update job history with transfer results
|
||||||
@@ -515,3 +687,141 @@ func (s *Scheduler) RunJobNow(jobID uint) error {
|
|||||||
go s.executeJob(jobID)
|
go s.executeJob(jobID)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// calculateFileHash computes an MD5 hash for the given file path
|
||||||
|
func calculateFileHash(filePath string) (string, error) {
|
||||||
|
file, err := os.Open(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("error opening file: %v", err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
hash := md5.New()
|
||||||
|
if _, err := io.Copy(hash, file); err != nil {
|
||||||
|
return "", fmt.Errorf("error calculating hash: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return hex.EncodeToString(hash.Sum(nil)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// getFileInfo retrieves file stats like size, creation time, and modification time
|
||||||
|
func getFileInfo(filePath string) (int64, time.Time, time.Time, error) {
|
||||||
|
info, err := os.Stat(filePath)
|
||||||
|
if err != nil {
|
||||||
|
return 0, time.Time{}, time.Time{}, fmt.Errorf("error getting file info: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
size := info.Size()
|
||||||
|
modTime := info.ModTime()
|
||||||
|
|
||||||
|
// Get creation time (this is platform-specific)
|
||||||
|
// For simplicity, we'll use modification time as a fallback
|
||||||
|
createTime := modTime
|
||||||
|
|
||||||
|
return size, createTime, modTime, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasFileBeenProcessed checks if a file with the same hash has been processed before
|
||||||
|
func (s *Scheduler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
|
||||||
|
if fileHash == "" {
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// First try to find by hash (most reliable)
|
||||||
|
metadata, err := s.db.GetFileMetadataByHash(fileHash)
|
||||||
|
if err == nil && metadata != nil {
|
||||||
|
return true, metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkFileProcessingHistory checks processing history for a given file
|
||||||
|
func (s *Scheduler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
|
||||||
|
// Try to find by job and filename
|
||||||
|
metadata, err := s.db.GetFileMetadataByJobAndName(jobID, fileName)
|
||||||
|
if err == nil && metadata != nil {
|
||||||
|
return metadata, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
|
||||||
|
}
|
||||||
|
|
||||||
|
// getRemoteFileInfo gets metadata for a remote file using rclone lsjson
|
||||||
|
func (s *Scheduler) getRemoteFileInfo(config *db.TransferConfig, file string) (int64, time.Time, time.Time, string, error) {
|
||||||
|
// Get rclone config path
|
||||||
|
configPath := s.db.GetConfigRclonePath(config)
|
||||||
|
|
||||||
|
// Construct the appropriate source path
|
||||||
|
var sourcePath string
|
||||||
|
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
|
||||||
|
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
|
||||||
|
if config.SourcePath != "" && config.SourcePath != "/" {
|
||||||
|
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use rclone lsjson to get file details
|
||||||
|
rclonePath := os.Getenv("RCLONE_PATH")
|
||||||
|
if rclonePath == "" {
|
||||||
|
rclonePath = "rclone"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Construct the full path to the file
|
||||||
|
fullPath := fmt.Sprintf("%s/%s", sourcePath, file)
|
||||||
|
|
||||||
|
// Run rclone lsjson command
|
||||||
|
args := []string{
|
||||||
|
"--config", configPath,
|
||||||
|
"lsjson",
|
||||||
|
"--hash",
|
||||||
|
fullPath,
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd := exec.Command(rclonePath, args...)
|
||||||
|
output, err := cmd.CombinedOutput()
|
||||||
|
if err != nil {
|
||||||
|
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error getting remote file info: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse the JSON output
|
||||||
|
var files []map[string]interface{}
|
||||||
|
if err := json.Unmarshal(output, &files); err != nil {
|
||||||
|
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("error parsing lsjson output: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(files) == 0 {
|
||||||
|
return 0, time.Time{}, time.Time{}, "", fmt.Errorf("file not found: %s", file)
|
||||||
|
}
|
||||||
|
|
||||||
|
fileInfo := files[0]
|
||||||
|
|
||||||
|
// Extract file size
|
||||||
|
var fileSize int64
|
||||||
|
if size, ok := fileInfo["Size"].(float64); ok {
|
||||||
|
fileSize = int64(size)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Extract modification time
|
||||||
|
modTime := time.Now()
|
||||||
|
if modTimeStr, ok := fileInfo["ModTime"].(string); ok {
|
||||||
|
if parsedTime, err := time.Parse(time.RFC3339, modTimeStr); err == nil {
|
||||||
|
modTime = parsedTime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create time is usually not available for remote files, so we'll use modTime
|
||||||
|
createTime := modTime
|
||||||
|
|
||||||
|
// Calculate hash if available
|
||||||
|
var md5Hash string
|
||||||
|
if hashes, ok := fileInfo["Hashes"].(map[string]interface{}); ok {
|
||||||
|
if md5, ok := hashes["md5"].(string); ok {
|
||||||
|
md5Hash = md5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fileSize, createTime, modTime, md5Hash, nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,378 @@
|
|||||||
|
package handlers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
|
"github.com/gin-gonic/gin"
|
||||||
|
"github.com/starfleetcptn/gomft/components"
|
||||||
|
"github.com/starfleetcptn/gomft/internal/db"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FileMetadataHandler handles displaying and searching file metadata
|
||||||
|
type FileMetadataHandler struct {
|
||||||
|
DB *db.DB
|
||||||
|
}
|
||||||
|
|
||||||
|
// Register registers the file metadata routes
|
||||||
|
func (h *FileMetadataHandler) Register(router *gin.RouterGroup) {
|
||||||
|
fileGroup := router.Group("/files")
|
||||||
|
|
||||||
|
fileGroup.GET("", h.ListFileMetadata)
|
||||||
|
fileGroup.GET("/:id", h.GetFileMetadataDetails)
|
||||||
|
fileGroup.GET("/job/:job_id", h.GetFileMetadataForJob)
|
||||||
|
fileGroup.GET("/search", h.SearchFileMetadata)
|
||||||
|
fileGroup.DELETE("/:id", h.DeleteFileMetadata)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListFileMetadata displays a list of file metadata with pagination and filtering options
|
||||||
|
func (h *FileMetadataHandler) ListFileMetadata(c *gin.Context) {
|
||||||
|
userID := c.GetUint("userID")
|
||||||
|
|
||||||
|
// Query parameters for pagination and filtering
|
||||||
|
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||||
|
if limit < 1 || limit > 100 {
|
||||||
|
limit = 50
|
||||||
|
}
|
||||||
|
|
||||||
|
status := c.Query("status")
|
||||||
|
jobIDStr := c.Query("job_id")
|
||||||
|
fileName := c.Query("filename")
|
||||||
|
|
||||||
|
// Base query
|
||||||
|
query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
|
||||||
|
|
||||||
|
// Apply filters
|
||||||
|
if jobIDStr != "" {
|
||||||
|
jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
|
||||||
|
query = query.Where("file_metadata.job_id = ?", jobID)
|
||||||
|
} else {
|
||||||
|
// Only show files from jobs created by the current user
|
||||||
|
query = query.Where("jobs.created_by = ?", userID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if status != "" {
|
||||||
|
query = query.Where("file_metadata.status = ?", status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fileName != "" {
|
||||||
|
query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count total records for pagination
|
||||||
|
var totalCount int64
|
||||||
|
query.Count(&totalCount)
|
||||||
|
|
||||||
|
// Retrieve file metadata with pagination
|
||||||
|
var fileMetadata []db.FileMetadata
|
||||||
|
offset := (page - 1) * limit
|
||||||
|
err := query.Preload("Job").Preload("Job.Config").
|
||||||
|
Order("file_metadata.processed_time DESC").
|
||||||
|
Offset(offset).Limit(limit).
|
||||||
|
Find(&fileMetadata).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create context for template
|
||||||
|
ctx := components.CreateTemplateContext(c)
|
||||||
|
|
||||||
|
// Render the file metadata list template
|
||||||
|
data := components.FileMetadataListData{
|
||||||
|
Files: fileMetadata,
|
||||||
|
TotalCount: totalCount,
|
||||||
|
Page: page,
|
||||||
|
Limit: limit,
|
||||||
|
TotalPages: int(totalCount) / limit,
|
||||||
|
Filter: components.FileMetadataFilter{
|
||||||
|
Status: status,
|
||||||
|
JobID: jobIDStr,
|
||||||
|
FileName: fileName,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// If total count is not exactly divisible by limit, add one more page
|
||||||
|
if int(totalCount)%limit > 0 {
|
||||||
|
data.TotalPages++
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Header("Content-Type", "text/html")
|
||||||
|
components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadataDetails displays detailed information about a specific file
|
||||||
|
func (h *FileMetadataHandler) GetFileMetadataDetails(c *gin.Context) {
|
||||||
|
userID := c.GetUint("userID")
|
||||||
|
|
||||||
|
// Get file ID from URL parameter
|
||||||
|
fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retrieve file metadata
|
||||||
|
var fileMetadata db.FileMetadata
|
||||||
|
err = h.DB.DB.Preload("Job").Preload("Job.Config").First(&fileMetadata, fileID).Error
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the user has access to this file (file must belong to a job created by the user)
|
||||||
|
var jobCreator uint
|
||||||
|
err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
|
||||||
|
if err != nil || jobCreator != userID {
|
||||||
|
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this file"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create context for template
|
||||||
|
ctx := components.CreateTemplateContext(c)
|
||||||
|
|
||||||
|
// Render the file metadata details template
|
||||||
|
data := components.FileMetadataDetailsData{
|
||||||
|
File: fileMetadata,
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Header("Content-Type", "text/html")
|
||||||
|
components.FileMetadataDetails(ctx, data).Render(ctx, c.Writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetFileMetadataForJob displays file metadata for a specific job
|
||||||
|
func (h *FileMetadataHandler) GetFileMetadataForJob(c *gin.Context) {
|
||||||
|
userID := c.GetUint("userID")
|
||||||
|
|
||||||
|
// Get job ID from URL parameter
|
||||||
|
jobID, err := strconv.ParseUint(c.Param("job_id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid job ID"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the user has access to this job
|
||||||
|
var job db.Job
|
||||||
|
err = h.DB.DB.Where("id = ?", jobID).First(&job).Error
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusNotFound, gin.H{"error": "Job not found"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if job.CreatedBy != userID {
|
||||||
|
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to view this job's files"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Query parameters for pagination
|
||||||
|
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||||
|
if limit < 1 || limit > 100 {
|
||||||
|
limit = 50
|
||||||
|
}
|
||||||
|
|
||||||
|
status := c.Query("status")
|
||||||
|
fileName := c.Query("filename")
|
||||||
|
|
||||||
|
// Base query
|
||||||
|
query := h.DB.DB.Model(&db.FileMetadata{}).Where("job_id = ?", jobID)
|
||||||
|
|
||||||
|
// Apply filters
|
||||||
|
if status != "" {
|
||||||
|
query = query.Where("status = ?", status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fileName != "" {
|
||||||
|
query = query.Where("file_name LIKE ?", "%"+fileName+"%")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count total records for pagination
|
||||||
|
var totalCount int64
|
||||||
|
query.Count(&totalCount)
|
||||||
|
|
||||||
|
// Retrieve file metadata with pagination
|
||||||
|
var fileMetadata []db.FileMetadata
|
||||||
|
offset := (page - 1) * limit
|
||||||
|
err = query.Preload("Job").Preload("Job.Config").
|
||||||
|
Order("processed_time DESC").
|
||||||
|
Offset(offset).Limit(limit).
|
||||||
|
Find(&fileMetadata).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create context for template
|
||||||
|
ctx := components.CreateTemplateContext(c)
|
||||||
|
|
||||||
|
// Render the file metadata list template
|
||||||
|
data := components.FileMetadataListData{
|
||||||
|
Files: fileMetadata,
|
||||||
|
TotalCount: totalCount,
|
||||||
|
Page: page,
|
||||||
|
Limit: limit,
|
||||||
|
TotalPages: int(totalCount) / limit,
|
||||||
|
Job: &job,
|
||||||
|
Filter: components.FileMetadataFilter{
|
||||||
|
Status: status,
|
||||||
|
JobID: strconv.FormatUint(uint64(job.ID), 10),
|
||||||
|
FileName: fileName,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// If total count is not exactly divisible by limit, add one more page
|
||||||
|
if int(totalCount)%limit > 0 {
|
||||||
|
data.TotalPages++
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Header("Content-Type", "text/html")
|
||||||
|
components.FileMetadataList(ctx, data).Render(ctx, c.Writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchFileMetadata searches file metadata based on various criteria
|
||||||
|
func (h *FileMetadataHandler) SearchFileMetadata(c *gin.Context) {
|
||||||
|
userID := c.GetUint("userID")
|
||||||
|
|
||||||
|
// Query parameters for search and pagination
|
||||||
|
page, _ := strconv.Atoi(c.DefaultQuery("page", "1"))
|
||||||
|
if page < 1 {
|
||||||
|
page = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
limit, _ := strconv.Atoi(c.DefaultQuery("limit", "50"))
|
||||||
|
if limit < 1 || limit > 100 {
|
||||||
|
limit = 50
|
||||||
|
}
|
||||||
|
|
||||||
|
status := c.Query("status")
|
||||||
|
jobIDStr := c.Query("job_id")
|
||||||
|
fileName := c.Query("filename")
|
||||||
|
hash := c.Query("hash")
|
||||||
|
startDate := c.Query("start_date")
|
||||||
|
endDate := c.Query("end_date")
|
||||||
|
|
||||||
|
// Base query
|
||||||
|
query := h.DB.DB.Model(&db.FileMetadata{}).Joins("JOIN jobs ON file_metadata.job_id = jobs.id")
|
||||||
|
|
||||||
|
// Apply filters
|
||||||
|
if jobIDStr != "" {
|
||||||
|
jobID, _ := strconv.ParseUint(jobIDStr, 10, 64)
|
||||||
|
query = query.Where("file_metadata.job_id = ?", jobID)
|
||||||
|
} else {
|
||||||
|
// Only show files from jobs created by the current user
|
||||||
|
query = query.Where("jobs.created_by = ?", userID)
|
||||||
|
}
|
||||||
|
|
||||||
|
if status != "" {
|
||||||
|
query = query.Where("file_metadata.status = ?", status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if fileName != "" {
|
||||||
|
query = query.Where("file_metadata.file_name LIKE ?", "%"+fileName+"%")
|
||||||
|
}
|
||||||
|
|
||||||
|
if hash != "" {
|
||||||
|
query = query.Where("file_metadata.file_hash = ?", hash)
|
||||||
|
}
|
||||||
|
|
||||||
|
if startDate != "" {
|
||||||
|
query = query.Where("file_metadata.processed_time >= ?", startDate)
|
||||||
|
}
|
||||||
|
|
||||||
|
if endDate != "" {
|
||||||
|
query = query.Where("file_metadata.processed_time <= ?", endDate+" 23:59:59")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Count total records for pagination
|
||||||
|
var totalCount int64
|
||||||
|
query.Count(&totalCount)
|
||||||
|
|
||||||
|
// Retrieve file metadata with pagination
|
||||||
|
var fileMetadata []db.FileMetadata
|
||||||
|
offset := (page - 1) * limit
|
||||||
|
err := query.Preload("Job").Preload("Job.Config").
|
||||||
|
Order("file_metadata.processed_time DESC").
|
||||||
|
Offset(offset).Limit(limit).
|
||||||
|
Find(&fileMetadata).Error
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to retrieve file metadata"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create context for template
|
||||||
|
ctx := components.CreateTemplateContext(c)
|
||||||
|
|
||||||
|
// Render the file metadata search template
|
||||||
|
data := components.FileMetadataSearchData{
|
||||||
|
Files: fileMetadata,
|
||||||
|
TotalCount: totalCount,
|
||||||
|
Page: page,
|
||||||
|
Limit: limit,
|
||||||
|
TotalPages: int(totalCount) / limit,
|
||||||
|
Filter: components.FileMetadataFilter{
|
||||||
|
Status: status,
|
||||||
|
JobID: jobIDStr,
|
||||||
|
FileName: fileName,
|
||||||
|
Hash: hash,
|
||||||
|
StartDate: startDate,
|
||||||
|
EndDate: endDate,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// If total count is not exactly divisible by limit, add one more page
|
||||||
|
if int(totalCount)%limit > 0 {
|
||||||
|
data.TotalPages++
|
||||||
|
}
|
||||||
|
|
||||||
|
c.Header("Content-Type", "text/html")
|
||||||
|
components.FileMetadataSearch(ctx, data).Render(ctx, c.Writer)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteFileMetadata deletes a file metadata record
|
||||||
|
func (h *FileMetadataHandler) DeleteFileMetadata(c *gin.Context) {
|
||||||
|
userID := c.GetUint("userID")
|
||||||
|
|
||||||
|
// Get file ID from URL parameter
|
||||||
|
fileID, err := strconv.ParseUint(c.Param("id"), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid file ID"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if the user has access to this file
|
||||||
|
var fileMetadata db.FileMetadata
|
||||||
|
err = h.DB.DB.Preload("Job").First(&fileMetadata, fileID).Error
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusNotFound, gin.H{"error": "File not found"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var jobCreator uint
|
||||||
|
err = h.DB.DB.Model(&db.Job{}).Where("id = ?", fileMetadata.JobID).Pluck("created_by", &jobCreator).Error
|
||||||
|
if err != nil || jobCreator != userID {
|
||||||
|
c.JSON(http.StatusForbidden, gin.H{"error": "You don't have permission to delete this file"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete the file metadata
|
||||||
|
err = h.DB.DeleteFileMetadata(uint(fileID))
|
||||||
|
if err != nil {
|
||||||
|
c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to delete file metadata"})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Redirect to the file list
|
||||||
|
c.Redirect(http.StatusFound, "/files")
|
||||||
|
}
|
||||||
@@ -4,8 +4,8 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"strconv"
|
||||||
|
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"github.com/starfleetcptn/gomft/components"
|
"github.com/starfleetcptn/gomft/components"
|
||||||
@@ -187,6 +187,10 @@ func (h *Handlers) RegisterRoutes(router *gin.Engine) {
|
|||||||
authorized.POST("/profile/theme", h.HandleUpdateTheme)
|
authorized.POST("/profile/theme", h.HandleUpdateTheme)
|
||||||
authorized.POST("/logout", h.HandleLogout)
|
authorized.POST("/logout", h.HandleLogout)
|
||||||
|
|
||||||
|
// File metadata routes
|
||||||
|
fileMetadataHandler := &FileMetadataHandler{DB: h.DB}
|
||||||
|
fileMetadataHandler.Register(authorized)
|
||||||
|
|
||||||
// AJAX routes for dashboard
|
// AJAX routes for dashboard
|
||||||
authorized.GET("/dashboard/data", h.HandleDashboardData)
|
authorized.GET("/dashboard/data", h.HandleDashboardData)
|
||||||
authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
|
authorized.GET("/dashboard/jobs", h.HandleDashboardJobsData)
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
|
||||||
// AuthMiddleware is a middleware function that checks if the request has a valid JWT token
|
// AuthMiddleware is a middleware function that checks if the request has a valid JWT token
|
||||||
func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
|
func (m *Middleware) AuthMiddleware() gin.HandlerFunc {
|
||||||
return func(c *gin.Context) {
|
return func(c *gin.Context) {
|
||||||
|
|||||||
Reference in New Issue
Block a user