feat: Implement multi-configuration support for jobs

- Add support for multiple transfer configurations per job, allowing users to select one or more configurations.
- Update job creation and editing forms to handle multiple configuration selections with checkboxes.
- Enhance job processing logic to iterate through all associated configurations during execution.
- Introduce database migrations to add necessary fields for storing multiple configuration IDs.
- Update dashboard and history views to display configuration details for jobs.
- Refactor related templates and handlers to accommodate the new multi-configuration functionality.
This commit is contained in:
StarFleetCPTN
2025-03-13 20:04:13 -07:00
parent d1967b4402
commit d6fa0c1603
11 changed files with 1054 additions and 528 deletions
+2 -1
View File
@@ -12,6 +12,7 @@ type DashboardData struct {
ActiveTransfers int
CompletedToday int
FailedTransfers int
Configs map[uint]db.TransferConfig
}
templ Dashboard(ctx context.Context, data DashboardData) {
@@ -113,7 +114,7 @@ templ Dashboard(ctx context.Context, data DashboardData) {
</div>
<div class="flex-1 min-w-0">
<p class="text-sm font-medium text-secondary-900 truncate dark:text-secondary-100">
{ job.Job.Config.Name }
{ getConfigNameForHistory(job, data.Configs) }
</p>
<div class="flex items-center mt-1">
<i class="fas fa-clock text-xs text-secondary-500 dark:text-secondary-400 mr-1"></i>
+24 -1
View File
@@ -13,6 +13,7 @@ type HistoryData struct {
SearchTerm string
PageSize int
Total int
Configs map[uint]db.TransferConfig // Map of config IDs to configs for quick lookup
}
// min returns the smaller of x or y
@@ -23,6 +24,28 @@ func min(x, y int) int {
return y
}
// getConfigNameForHistory returns the appropriate name for the config used in a job history entry
func getConfigNameForHistory(history db.JobHistory, configs map[uint]db.TransferConfig) string {
// If ConfigID is set in the history record, use that to get the config name
if history.ConfigID > 0 {
if config, exists := configs[history.ConfigID]; exists {
return config.Name
}
}
// Fallback to the Job's default Config if it exists
if history.Job.Config.ID > 0 {
return history.Job.Config.Name
}
// If we can't determine the config name, show a default with the job name
if history.Job.Name != "" {
return fmt.Sprintf("%s (unknown config)", history.Job.Name)
}
return "Unknown Configuration"
}
// HistoryContent renders only the content part of the history page for HTMX requests
templ HistoryContent(ctx context.Context, data HistoryData) {
if len(data.History) == 0 {
@@ -47,7 +70,7 @@ templ HistoryContent(ctx context.Context, data HistoryData) {
<div class="px-4 py-4 sm:px-6">
<div class="flex items-center justify-between">
<div class="flex items-center">
<p class="text-sm font-medium text-primary-600 dark:text-primary-400 truncate">{ history.Job.Config.Name }</p>
<p class="text-sm font-medium text-primary-600 dark:text-primary-400 truncate">{ getConfigNameForHistory(history, data.Configs) }</p>
if history.Status == "completed" {
<span class="ml-2 px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 dark:bg-green-900 text-green-800 dark:text-green-300">
Completed
+81 -71
View File
@@ -26,6 +26,18 @@ func getJobTitle(isNew bool) string {
return "Edit Job"
}
// configSelected checks if a config ID is selected for a job
func configSelected(job *db.Job, configID uint) bool {
// Check if the job has the config ID in its list
for _, id := range job.GetConfigIDsList() {
if id == configID {
return true
}
}
// As a fallback, check the primary ConfigID
return job.ConfigID == configID
}
templ JobForm(ctx context.Context, data JobFormData) {
@LayoutWithContext(getJobFormTitle(data.IsNew), ctx) {
<div class="min-h-[calc(100vh-4rem)] flex items-center justify-center py-12 px-4 sm:px-6 lg:px-8 bg-secondary-50 dark:bg-secondary-900">
@@ -47,11 +59,7 @@ templ JobForm(ctx context.Context, data JobFormData) {
class="space-y-6"
hx-post="/jobs"
hx-target="body"
hx-boost="true"
@htmx:before-request="loading = true"
@htmx:after-request="loading = false"
@htmx:response-error="$dispatch('notification', { message: 'Failed to create job: ' + event.detail.xhr.responseText, type: 'error' })"
x-data="{ name: '', configId: '', schedule: '', enabled: true, loading: false, validate() { return this.configId && this.schedule; } }">
hx-boost="true">
<div class="space-y-6">
<div>
<label for="name" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job Name</label>
@@ -63,7 +71,6 @@ templ JobForm(ctx context.Context, data JobFormData) {
type="text"
name="name"
id="name"
x-model="name"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Daily Production Backup"/>
</div>
@@ -74,23 +81,34 @@ templ JobForm(ctx context.Context, data JobFormData) {
</div>
<div>
<label for="config_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Transfer Configuration</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-cog text-secondary-400 dark:text-secondary-600"></i>
</div>
<select
id="config_id"
name="config_id"
x-model="configId"
required
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500">
<option value="">Select a configuration</option>
<label class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Transfer Configurations</label>
<div class="bg-secondary-50 dark:bg-secondary-800 border border-secondary-300 dark:border-secondary-700 rounded-lg max-h-60 overflow-y-auto">
if len(data.Configs) > 0 {
for _, config := range data.Configs {
<option value={ fmt.Sprint(config.ID) }>{ config.Name }</option>
}
</select>
<div class="flex items-center p-2 hover:bg-secondary-100 dark:hover:bg-secondary-700">
<input
type="checkbox"
id={ fmt.Sprintf("new-config-%d", config.ID) }
name="config_ids[]"
value={ fmt.Sprint(config.ID) }
class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"/>
<label
for={ fmt.Sprintf("new-config-%d", config.ID) }
class="ml-2 block text-sm text-secondary-700 dark:text-secondary-300 cursor-pointer w-full py-2">
{ config.Name }
</label>
</div>
}
} else {
<div class="text-center py-4 text-secondary-500 dark:text-secondary-400">
No configurations available. <a href="/configs/new" class="text-primary-600 hover:text-primary-500">Create one</a>
</div>
}
</div>
<p class="mt-2 text-sm text-secondary-500 dark:text-secondary-400">
<i class="fas fa-info-circle mr-1"></i>
Select one or more configurations to run on this schedule.
</p>
</div>
<div>
@@ -103,7 +121,6 @@ templ JobForm(ctx context.Context, data JobFormData) {
type="text"
name="schedule"
id="schedule"
x-model="schedule"
required
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="*/15 * * * *"/>
@@ -119,10 +136,10 @@ templ JobForm(ctx context.Context, data JobFormData) {
<input
type="checkbox"
id="enabled"
x-model="enabled"
class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"
checked/>
<input type="hidden" name="enabled" :value="enabled.toString()"/>
name="enabled"
value="true"
checked
class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"/>
<label for="enabled" class="ml-2 block text-sm font-medium text-secondary-700 dark:text-secondary-300">Enable this job</label>
</div>
<p class="mt-2 text-sm text-secondary-500 dark:text-secondary-400">
@@ -139,19 +156,9 @@ templ JobForm(ctx context.Context, data JobFormData) {
</a>
<button
type="submit"
class="btn-primary flex items-center justify-center px-4 py-2"
x-bind:disabled="!validate() || loading">
<span x-show="!loading" class="flex items-center">
class="btn-primary flex items-center justify-center px-4 py-2">
<i class="fas fa-plus mr-2"></i>
Create Job
</span>
<span x-show="loading" class="flex items-center">
<svg class="animate-spin -ml-1 mr-3 h-5 w-5 text-white" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
Processing...
</span>
</button>
</div>
</form>
@@ -160,11 +167,7 @@ templ JobForm(ctx context.Context, data JobFormData) {
class="space-y-6"
hx-post={ fmt.Sprintf("/jobs/%d", data.Job.ID) }
hx-target="body"
hx-boost="true"
@htmx:before-request="loading = true"
@htmx:after-request="loading = false"
@htmx:response-error="$dispatch('notification', { message: 'Failed to update job: ' + event.detail.xhr.responseText, type: 'error' })"
x-data={ fmt.Sprintf("{ name: '%s', configId: '%d', schedule: '%s', enabled: %v, loading: false, validate() { return this.configId && this.schedule; } }", data.Job.Name, data.Job.ConfigID, data.Job.Schedule, data.Job.Enabled) }>
hx-boost="true">
<div class="space-y-6">
<div>
<label for="name" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Job Name</label>
@@ -176,7 +179,7 @@ templ JobForm(ctx context.Context, data JobFormData) {
type="text"
name="name"
id="name"
x-model="name"
value={ data.Job.Name }
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Daily Production Backup"/>
</div>
@@ -187,23 +190,37 @@ templ JobForm(ctx context.Context, data JobFormData) {
</div>
<div>
<label for="config_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Transfer Configuration</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-cog text-secondary-400 dark:text-secondary-600"></i>
</div>
<select
id="config_id"
name="config_id"
x-model="configId"
required
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500">
<option value="">Select a configuration</option>
<label class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Transfer Configurations</label>
<div class="bg-secondary-50 dark:bg-secondary-800 border border-secondary-300 dark:border-secondary-700 rounded-lg max-h-60 overflow-y-auto">
if len(data.Configs) > 0 {
for _, config := range data.Configs {
<option value={ fmt.Sprint(config.ID) } if data.Job != nil && data.Job.ConfigID == config.ID { selected }>{ config.Name }</option>
<div class="flex items-center p-2 hover:bg-secondary-100 dark:hover:bg-secondary-700">
<input
type="checkbox"
id={ fmt.Sprintf("config-%d", config.ID) }
name="config_ids[]"
value={ fmt.Sprint(config.ID) }
if configSelected(data.Job, config.ID) {
checked
}
</select>
class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"/>
<label
for={ fmt.Sprintf("config-%d", config.ID) }
class="ml-2 block text-sm text-secondary-700 dark:text-secondary-300 cursor-pointer w-full py-2">
{ config.Name }
</label>
</div>
}
} else {
<div class="text-center py-4 text-secondary-500 dark:text-secondary-400">
No configurations available. <a href="/configs/new" class="text-primary-600 hover:text-primary-500">Create one</a>
</div>
}
</div>
<p class="mt-2 text-sm text-secondary-500 dark:text-secondary-400">
<i class="fas fa-info-circle mr-1"></i>
Select one or more configurations to run on this schedule.
</p>
</div>
<div>
@@ -216,7 +233,7 @@ templ JobForm(ctx context.Context, data JobFormData) {
type="text"
name="schedule"
id="schedule"
x-model="schedule"
value={ data.Job.Schedule }
required
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="*/15 * * * *"/>
@@ -232,9 +249,12 @@ templ JobForm(ctx context.Context, data JobFormData) {
<input
type="checkbox"
id="enabled"
x-model="enabled"
name="enabled"
value="true"
if data.Job.Enabled {
checked
}
class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"/>
<input type="hidden" name="enabled" :value="enabled.toString()"/>
<label for="enabled" class="ml-2 block text-sm font-medium text-secondary-700 dark:text-secondary-300">Enable this job</label>
</div>
<p class="mt-2 text-sm text-secondary-500 dark:text-secondary-400">
@@ -251,19 +271,9 @@ templ JobForm(ctx context.Context, data JobFormData) {
</a>
<button
type="submit"
class="btn-primary flex items-center justify-center px-4 py-2"
x-bind:disabled="!validate() || loading">
<span x-show="!loading" class="flex items-center">
class="btn-primary flex items-center justify-center px-4 py-2">
<i class="fas fa-save mr-2"></i>
Save Changes
</span>
<span x-show="loading" class="flex items-center">
<svg class="animate-spin -ml-1 mr-3 h-5 w-5 text-white" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24">
<circle class="opacity-25" cx="12" cy="12" r="10" stroke="currentColor" stroke-width="4"></circle>
<path class="opacity-75" fill="currentColor" d="M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"></path>
</svg>
Processing...
</span>
</button>
</div>
</form>
+12 -2
View File
@@ -71,6 +71,7 @@ script triggerJobDelete(dialogId string, jobID uint, jobName string) {
type JobsData struct {
Jobs []db.Job
ConfigCount map[uint]int // Maps job ID to number of configs
}
templ Jobs(ctx context.Context, data JobsData) {
@@ -334,8 +335,17 @@ templ Jobs(ctx context.Context, data JobsData) {
<div class="mt-2 sm:flex sm:justify-between">
<div class="sm:flex">
<p class="flex items-center text-sm text-secondary-500 dark:text-secondary-400">
<i class="fas fa-cog flex-shrink-0 mr-1.5 h-5 w-5 text-secondary-400 dark:text-secondary-500"></i>
Config: { job.Config.Name }
<i class="fas fa-cogs flex-shrink-0 mr-1.5 h-5 w-5 text-secondary-400 dark:text-secondary-500"></i>
Configs:
<span class="ml-1">
if count, ok := data.ConfigCount[job.ID]; ok && count > 1 {
{ fmt.Sprintf("%d configurations", count) }
} else if job.ConfigID > 0 {
{ job.Config.Name }
} else {
{ "None" }
}
</span>
</p>
<p class="mt-2 flex items-center text-sm text-secondary-500 dark:text-secondary-400 sm:mt-0 sm:ml-6">
<i class="fas fa-calendar-alt flex-shrink-0 mr-1.5 h-5 w-5 text-secondary-400 dark:text-secondary-500"></i>
+85
View File
@@ -5,6 +5,8 @@ import (
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/glebarez/sqlite"
@@ -115,6 +117,7 @@ type Job struct {
Name string `form:"name"`
ConfigID uint `gorm:"not null" form:"config_id"`
Config TransferConfig `gorm:"foreignkey:ConfigID"`
ConfigIDs string `gorm:"column:config_ids"` // Comma-separated list of config IDs
Schedule string `gorm:"not null" form:"schedule"`
Enabled bool `gorm:"default:true" form:"enabled"`
LastRun *time.Time
@@ -125,10 +128,64 @@ type Job struct {
UpdatedAt time.Time
}
// GetConfigIDsList returns the list of config IDs as integers
func (j *Job) GetConfigIDsList() []uint {
if j.ConfigIDs == "" {
// If ConfigIDs is empty but ConfigID is set, return that as the only ID
if j.ConfigID > 0 {
return []uint{j.ConfigID}
}
return []uint{}
}
// Split the comma-separated string
strIDs := strings.Split(j.ConfigIDs, ",")
ids := make([]uint, 0, len(strIDs))
// Convert each string to uint
for _, strID := range strIDs {
if id, err := strconv.ParseUint(strings.TrimSpace(strID), 10, 32); err == nil {
ids = append(ids, uint(id))
}
}
return ids
}
// SetConfigIDsList sets the config IDs from a slice of uint
func (j *Job) SetConfigIDsList(ids []uint) {
// Convert to strings
strIDs := make([]string, len(ids))
for i, id := range ids {
strIDs[i] = strconv.FormatUint(uint64(id), 10)
}
// Join with commas
j.ConfigIDs = strings.Join(strIDs, ",")
// If there's at least one ID, set ConfigID to the first one for backward compatibility
if len(ids) > 0 {
j.ConfigID = ids[0]
}
}
// GetConfigIDsAsStrings returns the list of config IDs as strings for template rendering
func (j *Job) GetConfigIDsAsStrings() []string {
ids := j.GetConfigIDsList()
strIDs := make([]string, len(ids))
for i, id := range ids {
strIDs[i] = fmt.Sprintf("'%d'", id)
}
return strIDs
}
type JobHistory struct {
ID uint `gorm:"primarykey"`
JobID uint `gorm:"not null"`
Job Job `gorm:"foreignkey:JobID"`
ConfigID uint `gorm:"default:0"` // The specific config ID this history entry is for
StartTime time.Time `gorm:"not null"`
EndTime *time.Time
Status string `gorm:"not null"`
@@ -142,6 +199,7 @@ type FileMetadata struct {
ID uint `gorm:"primarykey"`
JobID uint `gorm:"not null;index"`
Job Job `gorm:"foreignkey:JobID"`
ConfigID uint `gorm:"default:0"` // The specific config ID this file was processed with
FileName string `gorm:"not null"`
OriginalPath string `gorm:"not null"`
FileSize int64 `gorm:"not null"`
@@ -777,3 +835,30 @@ func (db *DB) GetActiveJobs() ([]Job, error) {
err := db.Preload("Config").Where("enabled = ?", true).Find(&jobs).Error
return jobs, err
}
// GetConfigsForJob returns all transfer configurations associated with a job
func (db *DB) GetConfigsForJob(jobID uint) ([]TransferConfig, error) {
var job Job
if err := db.First(&job, jobID).Error; err != nil {
return nil, err
}
// Get the list of config IDs
configIDs := job.GetConfigIDsList()
if len(configIDs) == 0 {
// If there are no IDs in the list but there is a configID, use that
if job.ConfigID > 0 {
configIDs = []uint{job.ConfigID}
} else {
return []TransferConfig{}, nil
}
}
// Fetch all configs
var configs []TransferConfig
if err := db.Where("id IN ?", configIDs).Find(&configs).Error; err != nil {
return nil, err
}
return configs, nil
}
@@ -0,0 +1,49 @@
package migrations
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// AddMultiConfigSupport adds support for multiple configurations per job
func AddMultiConfigSupport() *gormigrate.Migration {
return &gormigrate.Migration{
ID: "20250315_add_multi_config_support",
Migrate: func(tx *gorm.DB) error {
// Add config_ids column to jobs table
if err := tx.Exec("ALTER TABLE jobs ADD COLUMN config_ids TEXT").Error; err != nil {
return err
}
// Add config_id column to job_histories table
if err := tx.Exec("ALTER TABLE job_histories ADD COLUMN config_id INTEGER").Error; err != nil {
return err
}
// Add config_id column to file_metadata table
if err := tx.Exec("ALTER TABLE file_metadata ADD COLUMN config_id INTEGER").Error; err != nil {
return err
}
// Update existing jobs to set the config_ids field to match the current config_id
if err := tx.Exec("UPDATE jobs SET config_ids = config_id WHERE config_id > 0").Error; err != nil {
return err
}
return nil
},
Rollback: func(tx *gorm.DB) error {
// Drop the config_id columns from job_histories and file_metadata
if err := tx.Exec("ALTER TABLE job_histories DROP COLUMN config_id").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE file_metadata DROP COLUMN config_id").Error; err != nil {
return err
}
// Drop the config_ids column from jobs
return tx.Exec("ALTER TABLE jobs DROP COLUMN config_ids").Error
},
}
}
+1
View File
@@ -13,6 +13,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
AddCloudStorageFields(),
AddSkipProcessedFilesColumn(),
AddMaxConcurrentTransfersColumn(),
AddMultiConfigSupport(),
}
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
+155
View File
@@ -0,0 +1,155 @@
package handlers
import (
"errors"
"fmt"
"net/http"
"strconv"
"github.com/gorilla/mux"
"gorm.io/gorm"
"github.com/your-project/db"
)
// handleCreateJob handles the creation of a new job
func (h *Handler) handleCreateJob(w http.ResponseWriter, r *http.Request) {
// Parse form data
if err := r.ParseForm(); err != nil {
h.Logger.Error("Error parsing form: %v", err)
http.Error(w, "Error parsing form", http.StatusBadRequest)
return
}
// Get form values
name := r.FormValue("name")
schedule := r.FormValue("schedule")
enabled := r.FormValue("enabled")
// Get config IDs
configIDs := r.Form["config_ids[]"]
// Validate required fields
if len(configIDs) == 0 {
http.Error(w, "At least one configuration must be selected", http.StatusBadRequest)
return
}
if schedule == "" {
http.Error(w, "Schedule is required", http.StatusBadRequest)
return
}
// Parse config IDs and validate they exist
var configIDsList []uint
for _, configIDStr := range configIDs {
cID, err := strconv.ParseUint(configIDStr, 10, 32)
if err != nil {
h.Logger.Error("Error parsing config ID: %v", err)
http.Error(w, "Invalid config ID", http.StatusBadRequest)
return
}
// Validate config exists
var config db.TransferConfig
if err := h.DB.First(&config, cID).Error; err != nil {
h.Logger.Error("Config not found: %v", err)
http.Error(w, fmt.Sprintf("Config ID %d not found", cID), http.StatusBadRequest)
return
}
configIDsList = append(configIDsList, uint(cID))
}
// Create job with parsed values
job := db.Job{
Name: name,
Schedule: schedule,
Enabled: enabled == "true",
}
// Set config IDs
job.SetConfigIDsList(configIDsList)
// ... existing code ...
}
func (h *Handler) handleUpdateJob(w http.ResponseWriter, r *http.Request) {
// Parse path params
vars := mux.Vars(r)
jobID, err := strconv.ParseUint(vars["id"], 10, 32)
if err != nil {
h.Logger.Error("Error parsing job ID: %v", err)
http.Error(w, "Invalid job ID", http.StatusBadRequest)
return
}
// Get existing job
var job db.Job
if err := h.DB.First(&job, jobID).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
http.Error(w, "Job not found", http.StatusNotFound)
} else {
h.Logger.Error("Error getting job: %v", err)
http.Error(w, "Error getting job", http.StatusInternalServerError)
}
return
}
// Parse form data
if err := r.ParseForm(); err != nil {
h.Logger.Error("Error parsing form: %v", err)
http.Error(w, "Error parsing form", http.StatusBadRequest)
return
}
// Get form values
name := r.FormValue("name")
schedule := r.FormValue("schedule")
enabled := r.FormValue("enabled")
// Get config IDs
configIDs := r.Form["config_ids[]"]
// Validate required fields
if len(configIDs) == 0 {
http.Error(w, "At least one configuration must be selected", http.StatusBadRequest)
return
}
if schedule == "" {
http.Error(w, "Schedule is required", http.StatusBadRequest)
return
}
// Parse config IDs and validate they exist
var configIDsList []uint
for _, configIDStr := range configIDs {
cID, err := strconv.ParseUint(configIDStr, 10, 32)
if err != nil {
h.Logger.Error("Error parsing config ID: %v", err)
http.Error(w, "Invalid config ID", http.StatusBadRequest)
return
}
// Validate config exists
var config db.TransferConfig
if err := h.DB.First(&config, cID).Error; err != nil {
h.Logger.Error("Config not found: %v", err)
http.Error(w, fmt.Sprintf("Config ID %d not found", cID), http.StatusBadRequest)
return
}
configIDsList = append(configIDsList, uint(cID))
}
// Update job with parsed values
job.Name = name
job.Schedule = schedule
job.Enabled = enabled == "true"
// Set config IDs
job.SetConfigIDsList(configIDsList)
// ... existing code ...
}
+138 -125
View File
@@ -296,68 +296,82 @@ func (s *Scheduler) executeJob(jobID uint) {
// Get job details
var job db.Job
if err := s.db.Preload("Config").First(&job, jobID).Error; err != nil {
if err := s.db.First(&job, jobID).Error; err != nil {
s.log.LogError("Error loading job %d: %v", jobID, err)
return
}
if job.Config.ID == 0 {
s.log.LogError("Error: job %d has no associated config", jobID)
// Get all configurations associated with this job
configs, err := s.db.GetConfigsForJob(jobID)
if err != nil {
s.log.LogError("Error loading configurations for job %d: %v", jobID, err)
return
}
// Add explicit database reload of the config to ensure we have the latest values
var config db.TransferConfig
if err := s.db.First(&config, job.Config.ID).Error; err != nil {
s.log.LogError("Error loading config %d: %v", job.Config.ID, err)
if len(configs) == 0 {
s.log.LogError("Error: job %d has no associated configurations", jobID)
return
}
// Replace the job's config with the freshly loaded one
job.Config = config
// Now the rest of your code will use the correct value
s.log.LogInfo("Loaded job %d with config: source=%s:%s, dest=%s:%s, skipProcessedFiles=%v, maxConcurrentTransfers=%d",
jobID,
job.Config.SourceType,
job.Config.SourcePath,
job.Config.DestinationType,
job.Config.DestinationPath,
job.Config.SkipProcessedFiles,
job.Config.MaxConcurrentTransfers,
)
s.log.LogInfo("Loaded job %d with %d configurations", jobID, len(configs))
// Create job history entry
// Update job last run time
startTime := time.Now()
job.LastRun = &startTime
if err := s.db.UpdateJobStatus(&job); err != nil {
s.log.LogError("Error updating job last run time for job %d: %v", jobID, err)
}
// Process each configuration in sequence
for i, config := range configs {
// Create job history entry for this configuration
history := &db.JobHistory{
JobID: jobID,
StartTime: startTime,
ConfigID: config.ID,
StartTime: time.Now(),
Status: "running",
FilesTransferred: 0,
BytesTransferred: 0,
ErrorMessage: "",
}
if err := s.db.CreateJobHistory(history); err != nil {
s.log.LogError("Error creating job history for job %d: %v", jobID, err)
return
s.log.LogError("Error creating job history for job %d, config %d: %v", jobID, config.ID, err)
continue
}
// Update job last run time
job.LastRun = &history.StartTime
s.log.LogInfo("Processing configuration %d (%d/%d) for job %d: source=%s:%s, dest=%s:%s",
config.ID,
i+1,
len(configs),
jobID,
config.SourceType,
config.SourcePath,
config.DestinationType,
config.DestinationPath,
)
// Execute the configuration transfer
s.executeConfigTransfer(job, config, history)
}
// Update next run time if job is still scheduled
if entry := s.cron.Entry(s.jobs[jobID]); entry.ID != 0 {
job.NextRun = &entry.Next
if err := s.db.UpdateJobStatus(&job); err != nil {
s.log.LogError("Error updating job last run time for job %d: %v", jobID, err)
s.log.LogError("Error updating next run time for job %d: %v", jobID, err)
} else {
s.log.LogInfo("Next run time for job %d: %s", jobID, entry.Next.Format(time.RFC3339))
}
}
// Reload the job from the database to get the latest values
if err := s.db.Preload("Config").First(&job, jobID).Error; err != nil {
s.log.LogError("Error reloading job %d: %v", jobID, err)
return
}
// executeConfigTransfer performs the actual file transfer for a single configuration
func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, history *db.JobHistory) {
// Track files already processed in this job execution to prevent duplicates
processedFiles := make(map[string]bool)
// Get rclone config path
configPath := s.db.GetConfigRclonePath(&job.Config)
configPath := s.db.GetConfigRclonePath(&config)
// Use lsjson to get file list and metadata in one operation instead of separate size and ls commands
listArgs := []string{
@@ -368,17 +382,17 @@ func (s *Scheduler) executeJob(jobID uint) {
}
// Add file pattern filter if specified
if job.Config.FilePattern != "" && job.Config.FilePattern != "*" {
if config.FilePattern != "" && config.FilePattern != "*" {
// Create a temporary filter file for complex patterns
filterFile, err := createRcloneFilterFile(job.Config.FilePattern)
filterFile, err := createRcloneFilterFile(config.FilePattern)
if err != nil {
s.log.LogError("Error creating filter file for job %d: %v", jobID, err)
s.log.LogError("Error creating filter file for job %d, config %d: %v", job.ID, config.ID, err)
history.Status = "failed"
history.ErrorMessage = fmt.Sprintf("Filter Creation Error: %v", err)
endTime := time.Now()
history.EndTime = &endTime
if err := s.db.UpdateJobHistory(history); err != nil {
s.log.LogError("Error updating job history for job %d: %v", jobID, err)
s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
}
return
}
@@ -388,19 +402,19 @@ func (s *Scheduler) executeJob(jobID uint) {
// Add source path with bucket for S3-compatible storage
var sourceListPath string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
sourceListPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourceBucket)
if job.Config.SourcePath != "" && job.Config.SourcePath != "/" {
sourceListPath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath)
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
sourceListPath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
if config.SourcePath != "" && config.SourcePath != "/" {
sourceListPath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
}
} else {
sourceListPath = fmt.Sprintf("source_%d:%s", job.Config.ID, job.Config.SourcePath)
sourceListPath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
}
listArgs = append(listArgs, sourceListPath)
// Execute lsjson command
s.log.LogInfo("Listing files with metadata for job %d: rclone %s", jobID, strings.Join(listArgs, " "))
s.log.LogInfo("Listing files with metadata for job %d, config %d: rclone %s", job.ID, config.ID, strings.Join(listArgs, " "))
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
rclonePath = "rclone"
@@ -409,14 +423,14 @@ func (s *Scheduler) executeJob(jobID uint) {
listOutput, listErr := listCmd.CombinedOutput()
if listErr != nil {
s.log.LogError("Error listing files for job %d: %v", jobID, listErr)
s.log.LogError("Error listing files for job %d, config %d: %v", job.ID, config.ID, listErr)
// s.log.Debug.Printf("Output: %s", string(listOutput))
history.Status = "failed"
history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
endTime := time.Now()
history.EndTime = &endTime
if err := s.db.UpdateJobHistory(history); err != nil {
s.log.LogError("Error updating job history for job %d: %v", jobID, err)
s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
}
return
}
@@ -424,13 +438,13 @@ func (s *Scheduler) executeJob(jobID uint) {
// Parse JSON output to get file information
var fileEntries []map[string]interface{}
if err := json.Unmarshal(listOutput, &fileEntries); err != nil {
s.log.LogError("Error parsing file list JSON for job %d: %v", jobID, err)
s.log.LogError("Error parsing file list JSON for job %d, config %d: %v", job.ID, config.ID, err)
history.Status = "failed"
history.ErrorMessage = fmt.Sprintf("JSON Parsing Error: %v", err)
endTime := time.Now()
history.EndTime = &endTime
if err := s.db.UpdateJobHistory(history); err != nil {
s.log.LogError("Error updating job history for job %d: %v", jobID, err)
s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
}
return
}
@@ -453,17 +467,24 @@ func (s *Scheduler) executeJob(jobID uint) {
}
}
s.log.LogInfo("Found %d files totaling %d bytes to transfer for job %d", len(files), totalSize, jobID)
s.log.LogInfo("Found %d files totaling %d bytes to transfer for job %d, config %d", len(files), totalSize, job.ID, config.ID)
// Update history with size information
history.BytesTransferred = totalSize
if len(files) == 0 {
s.log.LogInfo("No files to transfer for job %d", jobID)
s.log.LogInfo("No files to transfer for job %d, config %d", job.ID, config.ID)
history.Status = "completed"
history.ErrorMessage = ""
history.FilesTransferred = 0
} else {
endTime := time.Now()
history.EndTime = &endTime
if err := s.db.UpdateJobHistory(history); err != nil {
s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
}
return
}
var transferErrors []string
filesTransferred := 0
@@ -471,11 +492,11 @@ func (s *Scheduler) executeJob(jobID uint) {
var mutex sync.Mutex
// Determine number of concurrent transfers
maxConcurrent := job.Config.MaxConcurrentTransfers
maxConcurrent := config.MaxConcurrentTransfers
if maxConcurrent < 1 {
maxConcurrent = 1 // Default to 1 if not set
}
s.log.LogInfo("Using %d concurrent transfers for job %d", maxConcurrent, jobID)
s.log.LogInfo("Using %d concurrent transfers for job %d, config %d", maxConcurrent, job.ID, config.ID)
// Create wait group for concurrent processing
var wg sync.WaitGroup
@@ -498,11 +519,12 @@ func (s *Scheduler) executeJob(jobID uint) {
// Extract hash from the file entry
fileHash := ""
if hash, ok := fileEntry["Hashes"].(map[string]interface{}); ok {
if hashes, ok := fileEntry["Hashes"].(map[string]interface{}); ok {
// Try several hash algorithms in order of preference
for _, hashType := range []string{"SHA-1", "MD5"} {
if hashValue, found := hash[hashType]; found {
if hashStr, ok := hashValue.(string); ok {
for _, hashType := range []string{"SHA-1", "sha1", "MD5", "md5", "sha256", "crc32"} {
if hashValue, found := hashes[hashType]; found {
if hashStr, ok := hashValue.(string); ok && hashStr != "" {
s.log.LogDebug("Found hash %s: %s for file %s", hashType, hashStr, fileName)
fileHash = hashStr
break
}
@@ -510,6 +532,11 @@ func (s *Scheduler) executeJob(jobID uint) {
}
}
// Log if no hash was found
if fileHash == "" {
s.log.LogDebug("No hash found for file %s. Available fields: %v", fileName, fileEntry)
}
// Extract size from the file entry
fileSize := int64(0)
if size, ok := fileEntry["Size"].(float64); ok {
@@ -517,9 +544,9 @@ func (s *Scheduler) executeJob(jobID uint) {
}
// Skip files that have already been processed based on hash
skipFiles := job.Config.SkipProcessedFiles
skipFiles := config.SkipProcessedFiles
if skipFiles && fileHash != "" {
alreadyProcessed, prevMetadata, err := s.hasFileBeenProcessed(jobID, fileHash)
alreadyProcessed, prevMetadata, err := s.hasFileBeenProcessed(job.ID, fileHash)
if err == nil && alreadyProcessed {
s.log.LogDebug("File %s with hash %s was previously processed on %s with status: %s",
fileName, fileHash, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
@@ -543,7 +570,7 @@ func (s *Scheduler) executeJob(jobID uint) {
}
// Also check the processing history for this specific file name
prevMetadata, histErr := s.checkFileProcessingHistory(jobID, fileName)
prevMetadata, histErr := s.checkFileProcessingHistory(job.ID, fileName)
if histErr == nil {
s.log.LogDebug("File %s was previously processed on %s with status: %s",
fileName, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
@@ -592,6 +619,9 @@ func (s *Scheduler) executeJob(jobID uint) {
currentCreateTime := createTime
currentModTime := modTime
// Log the file information that will be processed
s.log.LogDebug("Processing file: %s, size: %d, hash: %s", currentFileName, currentFileSize, currentFileHash)
// Start goroutine for concurrent processing
go func() {
// Acquire semaphore
@@ -616,56 +646,56 @@ func (s *Scheduler) executeJob(jobID uint) {
var sourcePath, destPath string
// For S3, MinIO, and B2, include the bucket in the path
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourceBucket, currentFileName)
if job.Config.SourcePath != "" && job.Config.SourcePath != "/" {
sourcePath = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.SourcePath, currentFileName)
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, currentFileName)
if config.SourcePath != "" && config.SourcePath != "/" {
sourcePath = fmt.Sprintf("source_%d:%s/%s/%s", config.ID, config.SourceBucket, config.SourcePath, currentFileName)
}
} else {
sourcePath = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.SourcePath, currentFileName)
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourcePath, currentFileName)
}
var destFile string = currentFileName
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, currentFileName)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, currentFileName)
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, currentFileName)
if config.DestinationPath != "" && config.DestinationPath != "/" {
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", config.ID, config.DestBucket, config.DestinationPath, currentFileName)
}
} else {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, currentFileName)
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestinationPath, currentFileName)
}
// Add output filename pattern if specified
if job.Config.OutputPattern != "" {
if config.OutputPattern != "" {
// Process the output pattern for this specific file
destFile = ProcessOutputPattern(job.Config.OutputPattern, currentFileName)
destFile = ProcessOutputPattern(config.OutputPattern, currentFileName)
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestBucket, destFile)
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", job.Config.ID, job.Config.DestBucket, job.Config.DestinationPath, destFile)
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, destFile)
if config.DestinationPath != "" && config.DestinationPath != "/" {
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", config.ID, config.DestBucket, config.DestinationPath, destFile)
}
} else {
destPath = fmt.Sprintf("dest_%d:%s/%s", job.Config.ID, job.Config.DestinationPath, destFile)
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestinationPath, destFile)
}
s.log.LogDebug("Renaming file from %s to %s for job %d", currentFileName, destFile, jobID)
s.log.LogDebug("Renaming file from %s to %s for job %d, config %d", currentFileName, destFile, job.ID, config.ID)
}
// Add custom flags if specified
if job.Config.RcloneFlags != "" {
customFlags := strings.Split(job.Config.RcloneFlags, " ")
if config.RcloneFlags != "" {
customFlags := strings.Split(config.RcloneFlags, " ")
transferArgs = append(transferArgs, customFlags...)
s.log.LogDebug("Added custom flags for job %d: %v", jobID, customFlags)
s.log.LogDebug("Added custom flags for job %d, config %d: %v", job.ID, config.ID, customFlags)
}
// Add source and destination to the command
transferArgs = append(transferArgs, sourcePath, destPath)
// Execute transfer for this file
s.log.LogInfo("Executing rclone transfer command for job %d, file %s: rclone %s",
jobID, currentFileName, strings.Join(transferArgs, " "))
s.log.LogInfo("Executing rclone transfer command for job %d, config %d, file %s: rclone %s",
job.ID, config.ID, currentFileName, strings.Join(transferArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -684,7 +714,7 @@ func (s *Scheduler) executeJob(jobID uint) {
// Check if file was successfully transferred
if fileErr != nil {
s.log.LogError("Error transferring file %s for job %d: %v", currentFileName, jobID, fileErr)
s.log.LogError("Error transferring file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, fileErr)
mutex.Lock()
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", currentFileName, fileErr))
mutex.Unlock()
@@ -694,27 +724,27 @@ func (s *Scheduler) executeJob(jobID uint) {
mutex.Lock()
filesTransferred++
mutex.Unlock()
s.log.LogInfo("Successfully transferred file %s for job %d", currentFileName, jobID)
s.log.LogInfo("Successfully transferred file %s for job %d, config %d", currentFileName, job.ID, config.ID)
// Extract the actual destination path (without rclone remote prefix)
if job.Config.DestinationType == "local" {
destPathForDB = filepath.Join(job.Config.DestinationPath, destFile)
if config.DestinationType == "local" {
destPathForDB = filepath.Join(config.DestinationPath, destFile)
} else {
// For remote destinations, store the path format
if job.Config.DestinationType == "s3" || job.Config.DestinationType == "minio" || job.Config.DestinationType == "b2" {
if job.Config.DestinationPath != "" && job.Config.DestinationPath != "/" {
destPathForDB = fmt.Sprintf("%s/%s/%s", job.Config.DestBucket, job.Config.DestinationPath, destFile)
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
if config.DestinationPath != "" && config.DestinationPath != "/" {
destPathForDB = fmt.Sprintf("%s/%s/%s", config.DestBucket, config.DestinationPath, destFile)
} else {
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestBucket, destFile)
destPathForDB = fmt.Sprintf("%s/%s", config.DestBucket, destFile)
}
} else {
destPathForDB = fmt.Sprintf("%s/%s", job.Config.DestinationPath, destFile)
destPathForDB = fmt.Sprintf("%s/%s", config.DestinationPath, destFile)
}
}
// If archiving is enabled and transfer was successful, move files to archive
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
s.log.LogInfo("Archiving file %s for job %d", currentFileName, jobID)
if config.ArchiveEnabled && config.ArchivePath != "" {
s.log.LogInfo("Archiving file %s for job %d, config %d", currentFileName, job.ID, config.ID)
// We don't need to move the file since we used moveto, but we can copy it to archive
archiveArgs := []string{
@@ -725,16 +755,16 @@ func (s *Scheduler) executeJob(jobID uint) {
// Construct archive path with bucket if needed
var archiveDest string
if job.Config.SourceType == "s3" || job.Config.SourceType == "minio" || job.Config.SourceType == "b2" {
archiveDest = fmt.Sprintf("source_%d:%s/%s/%s", job.Config.ID, job.Config.SourceBucket, job.Config.ArchivePath, currentFileName)
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
archiveDest = fmt.Sprintf("source_%d:%s/%s/%s", config.ID, config.SourceBucket, config.ArchivePath, currentFileName)
} else {
archiveDest = fmt.Sprintf("source_%d:%s/%s", job.Config.ID, job.Config.ArchivePath, currentFileName)
archiveDest = fmt.Sprintf("source_%d:%s/%s", config.ID, config.ArchivePath, currentFileName)
}
archiveArgs = append(archiveArgs, archiveDest)
s.log.LogInfo("Executing rclone archive command for job %d, file %s: rclone %s",
jobID, currentFileName, strings.Join(archiveArgs, " "))
s.log.LogInfo("Executing rclone archive command for job %d, config %d, file %s: rclone %s",
job.ID, config.ID, currentFileName, strings.Join(archiveArgs, " "))
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
@@ -748,7 +778,7 @@ func (s *Scheduler) executeJob(jobID uint) {
// Check if file was successfully transferred
if archiveErr != nil {
s.log.LogError("Warning: Error archiving file %s for job %d: %v", currentFileName, jobID, archiveErr)
s.log.LogError("Warning: Error archiving file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, archiveErr)
mutex.Lock()
transferErrors = append(transferErrors,
fmt.Sprintf("Archive error for file %s: %v", currentFileName, archiveErr))
@@ -758,8 +788,8 @@ func (s *Scheduler) executeJob(jobID uint) {
}
}
if job.Config.DeleteAfterTransfer {
s.log.LogInfo("Deleting file %s for job %d", currentFileName, jobID)
if config.DeleteAfterTransfer {
s.log.LogInfo("Deleting file %s for job %d, config %d", currentFileName, job.ID, config.ID)
deleteArgs := []string{
"--config", configPath,
"deletefile",
@@ -768,7 +798,7 @@ func (s *Scheduler) executeJob(jobID uint) {
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
s.log.LogDebug("Output for file %s: %s", currentFileName, string(deleteOutput))
if deleteErr != nil {
s.log.LogError("Error deleting file %s for job %d: %v", currentFileName, jobID, deleteErr)
s.log.LogError("Error deleting file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, deleteErr)
mutex.Lock()
transferErrors = append(transferErrors,
fmt.Sprintf("Delete error for file %s: %v", currentFileName, deleteErr))
@@ -785,9 +815,10 @@ func (s *Scheduler) executeJob(jobID uint) {
// Create and save file metadata
metadata := &db.FileMetadata{
JobID: jobID,
JobID: job.ID,
ConfigID: config.ID,
FileName: currentFileName,
OriginalPath: job.Config.SourcePath,
OriginalPath: config.SourcePath,
FileSize: currentFileSize,
FileHash: currentFileHash,
CreationTime: currentCreateTime,
@@ -801,7 +832,7 @@ func (s *Scheduler) executeJob(jobID uint) {
if err := s.db.CreateFileMetadata(metadata); err != nil {
s.log.LogError("Error creating file metadata for %s: %v", currentFileName, err)
} else {
s.log.LogDebug("Created file metadata record for %s (ID: %d)", currentFileName, metadata.ID)
s.log.LogDebug("Created file metadata record for %s (ID: %d) with hash: %s", currentFileName, metadata.ID, currentFileHash)
}
}()
}
@@ -819,34 +850,16 @@ func (s *Scheduler) executeJob(jobID uint) {
history.Status = "completed_with_errors"
history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
len(transferErrors), strings.Join(transferErrors, "\n"))
}
} else {
history.Status = "completed"
}
// Update job history with completion status and end time
endTime := time.Now()
history.EndTime = &endTime
if job.Config.ArchiveEnabled && job.Config.ArchivePath != "" {
if history.ErrorMessage != "" {
history.Status = "completed_with_archive_error"
} else {
history.Status = "completed"
}
} else {
history.Status = "completed"
}
if err := s.db.UpdateJobHistory(history); err != nil {
s.log.LogError("Error updating job history for job %d: %v", jobID, err)
}
// Update next run time if job is still scheduled
if entry := s.cron.Entry(s.jobs[jobID]); entry.ID != 0 {
job.NextRun = &entry.Next
if err := s.db.UpdateJobStatus(&job); err != nil {
s.log.LogError("Error updating next run time for job %d: %v", jobID, err)
} else {
s.log.LogInfo("Next run time for job %d: %s", jobID, entry.Next.Format(time.RFC3339))
}
s.log.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
}
}
+79 -1
View File
@@ -17,7 +17,7 @@ func (h *Handlers) HandleDashboard(c *gin.Context) {
// Get recent job history
var recentHistory []db.JobHistory
h.DB.Order("start_time DESC").Limit(5).Find(&recentHistory)
h.DB.Preload("Job.Config").Order("start_time DESC").Limit(5).Find(&recentHistory)
// Get job statistics
var totalJobs int64
@@ -29,11 +29,50 @@ func (h *Handlers) HandleDashboard(c *gin.Context) {
var failedJobs int64
h.DB.Model(&db.JobHistory{}).Where("status = ?", "failed").Count(&failedJobs)
// Create a map to hold all relevant config IDs
configIDs := make(map[uint]bool)
// Collect all config IDs from recent history entries
for _, h := range recentHistory {
// Add the specific config ID used for this history entry if it exists
if h.ConfigID > 0 {
configIDs[h.ConfigID] = true
}
// Add the job's default config ID as a fallback
if h.Job.ConfigID > 0 {
configIDs[h.Job.ConfigID] = true
}
}
// Create a map to store all configs by their ID
configsMap := make(map[uint]db.TransferConfig)
// Load all necessary configurations
if len(configIDs) > 0 {
var configsList []db.TransferConfig
configIDsList := make([]uint, 0, len(configIDs))
// Extract config IDs from the map
for id := range configIDs {
configIDsList = append(configIDsList, id)
}
// Load all configurations in one query
if err := h.DB.Where("id IN ?", configIDsList).Find(&configsList).Error; err == nil {
// Create the lookup map
for _, config := range configsList {
configsMap[config.ID] = config
}
}
}
data := components.DashboardData{
RecentJobs: recentHistory,
ActiveTransfers: int(totalJobs),
CompletedToday: int(completedJobs),
FailedTransfers: int(failedJobs),
Configs: configsMap,
}
components.Dashboard(components.CreateTemplateContext(c), data).Render(c, c.Writer)
@@ -110,6 +149,44 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
return
}
// Create a map to hold all relevant config IDs
configIDs := make(map[uint]bool)
// Collect all config IDs from history entries
for _, h := range history {
// Add the specific config ID used for this history entry if it exists
if h.ConfigID > 0 {
configIDs[h.ConfigID] = true
}
// Add the job's default config ID as a fallback
if h.Job.ConfigID > 0 {
configIDs[h.Job.ConfigID] = true
}
}
// Create a map to store all configs by their ID
configsMap := make(map[uint]db.TransferConfig)
// Load all necessary configurations
if len(configIDs) > 0 {
var configsList []db.TransferConfig
configIDsList := make([]uint, 0, len(configIDs))
// Extract config IDs from the map
for id := range configIDs {
configIDsList = append(configIDsList, id)
}
// Load all configurations in one query
if err := h.DB.Where("id IN ?", configIDsList).Find(&configsList).Error; err == nil {
// Create the lookup map
for _, config := range configsList {
configsMap[config.ID] = config
}
}
}
data := components.HistoryData{
History: history,
CurrentPage: page,
@@ -117,6 +194,7 @@ func (h *Handlers) HandleHistory(c *gin.Context) {
SearchTerm: searchTerm,
PageSize: pageSize,
Total: int(total),
Configs: configsMap,
}
// If this is an HTMX request, only render the history content component
+110 -9
View File
@@ -3,6 +3,7 @@ package handlers
import (
"fmt"
"net/http"
"strconv"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/components"
@@ -16,8 +17,24 @@ func (h *Handlers) HandleJobs(c *gin.Context) {
var jobs []db.Job
h.DB.Where("created_by = ?", userID).Preload("Config").Find(&jobs)
// Create a map to store config counts for each job
configCount := make(map[uint]int)
// Count configurations for each job
for _, job := range jobs {
// Get all configurations for this job
configs, err := h.DB.GetConfigsForJob(job.ID)
if err != nil {
c.Error(fmt.Errorf("error loading configurations for job %d: %v", job.ID, err))
configCount[job.ID] = 0
} else {
configCount[job.ID] = len(configs)
}
}
data := components.JobsData{
Jobs: jobs,
ConfigCount: configCount,
}
components.Jobs(c, data).Render(c, c.Writer)
}
@@ -49,7 +66,16 @@ func (h *Handlers) HandleJobRunDetails(c *gin.Context) {
// Get the config
var config db.TransferConfig
if err := h.DB.First(&config, job.ConfigID).Error; err != nil {
// First try to get the specific config used in this job history record
configID := jobHistory.ConfigID
// If no ConfigID is set in the history, fall back to the job's primary ConfigID
if configID == 0 {
configID = job.ConfigID
}
if err := h.DB.First(&config, configID).Error; err != nil {
c.String(http.StatusNotFound, "Configuration not found")
return
}
@@ -113,17 +139,57 @@ func (h *Handlers) HandleEditJob(c *gin.Context) {
// HandleCreateJob handles the POST /jobs route
func (h *Handlers) HandleCreateJob(c *gin.Context) {
userID := c.GetUint("userID")
// Parse form data
var job db.Job
if err := c.ShouldBind(&job); err != nil {
c.String(http.StatusBadRequest, "Invalid form data")
return
}
userID := c.GetUint("userID")
job.CreatedBy = userID
// Get multiple config IDs from form
configIDs := c.PostFormArray("config_ids[]")
if len(configIDs) == 0 {
c.String(http.StatusBadRequest, "At least one configuration must be selected")
return
}
// Process config IDs
var configIDsList []uint
for _, configIDStr := range configIDs {
configID, err := strconv.ParseUint(configIDStr, 10, 32)
if err != nil {
c.String(http.StatusBadRequest, "Invalid configuration ID format")
return
}
// Verify that the config exists and belongs to the user
var config db.TransferConfig
if err := h.DB.First(&config, configID).Error; err != nil {
c.String(http.StatusBadRequest, "Invalid configuration selected")
return
}
// Check if the config belongs to the user
if config.CreatedBy != userID {
// Check if user is admin
isAdmin, exists := c.Get("isAdmin")
if !exists || isAdmin != true {
c.String(http.StatusForbidden, "You do not have permission to use this configuration")
return
}
}
configIDsList = append(configIDsList, uint(configID))
}
// Set the first config ID for backward compatibility
if len(configIDsList) > 0 {
job.ConfigID = configIDsList[0]
// Verify that the config exists and belongs to the user (using the first config as primary)
var config db.TransferConfig
if err := h.DB.First(&config, job.ConfigID).Error; err != nil {
c.String(http.StatusBadRequest, "Invalid configuration selected")
return
@@ -139,10 +205,17 @@ func (h *Handlers) HandleCreateJob(c *gin.Context) {
}
}
// If job name is empty, use the config name
// If job name is empty, use the primary config name
if job.Name == "" {
job.Name = config.Name
}
}
// Set the config IDs list
job.SetConfigIDsList(configIDsList)
// Set created by user
job.CreatedBy = userID
// Clear the Config field to prevent GORM from creating a new config
job.Config = db.TransferConfig{}
@@ -186,15 +259,31 @@ func (h *Handlers) HandleUpdateJob(c *gin.Context) {
// Get the old job values for comparison
oldJob := job
// Bind form data to job
// Parse form data
if err := c.ShouldBind(&job); err != nil {
c.String(http.StatusBadRequest, "Invalid form data")
return
}
// Verify that the config exists and belongs to the user
// Get multiple config IDs from form
configIDs := c.PostFormArray("config_ids[]")
if len(configIDs) == 0 {
c.String(http.StatusBadRequest, "At least one configuration must be selected")
return
}
// Process config IDs
var configIDsList []uint
for _, configIDStr := range configIDs {
configID, err := strconv.ParseUint(configIDStr, 10, 32)
if err != nil {
c.String(http.StatusBadRequest, "Invalid configuration ID format")
return
}
// Verify that the config exists
var config db.TransferConfig
if err := h.DB.First(&config, job.ConfigID).Error; err != nil {
if err := h.DB.First(&config, configID).Error; err != nil {
c.String(http.StatusBadRequest, "Invalid configuration selected")
return
}
@@ -209,10 +298,22 @@ func (h *Handlers) HandleUpdateJob(c *gin.Context) {
}
}
// If job name is empty, use the config name
if job.Name == "" {
configIDsList = append(configIDsList, uint(configID))
}
// Set the first config ID for backward compatibility
if len(configIDsList) > 0 {
job.ConfigID = configIDsList[0]
// If job name is empty, use the primary config name
var config db.TransferConfig
if err := h.DB.First(&config, job.ConfigID).Error; err == nil && job.Name == "" {
job.Name = config.Name
}
}
// Set the config IDs list
job.SetConfigIDsList(configIDsList)
// Preserve fields that shouldn't be updated
job.CreatedBy = oldJob.CreatedBy