mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-09-08 23:50:48 +02:00
refactor: job scheduling and execution components
- Refactored existing scheduler logic to integrate new components and improve job management.
This commit is contained in:
StarFleetCPTN
@@ -238,8 +238,6 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta
|
||||
}
|
||||
log.Printf("Successfully created temp config section for %s", remoteName)
|
||||
|
||||
// Declarations moved before the switch statement
|
||||
|
||||
RunLsd: // Label to jump to for local type
|
||||
|
||||
// --- Execute `rclone lsd` using the temporary config ---
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/robfig/cron/v3"
|
||||
"github.com/starfleetcptn/gomft/internal/db"
|
||||
)
|
||||
|
||||
// JobExecutor handles the execution logic for a single job run.
|
||||
type JobExecutor struct {
|
||||
db *db.DB
|
||||
logger *Logger
|
||||
cron *cron.Cron
|
||||
jobs map[uint]cron.EntryID // Shared map from Scheduler
|
||||
jobMutex *sync.Mutex // Shared mutex from Scheduler
|
||||
transferExecutor *TransferExecutor // TransferExecutor component
|
||||
notifier *Notifier // Notifier component
|
||||
// metadataHandler *MetadataHandler // Placeholder if needed directly
|
||||
}
|
||||
|
||||
// NewJobExecutor creates a new JobExecutor.
|
||||
func NewJobExecutor(
|
||||
database *db.DB,
|
||||
logger *Logger,
|
||||
cron *cron.Cron,
|
||||
jobsMap map[uint]cron.EntryID,
|
||||
jobMutex *sync.Mutex,
|
||||
transferExec *TransferExecutor,
|
||||
notify *Notifier,
|
||||
// metadata *MetadataHandler,
|
||||
) *JobExecutor {
|
||||
return &JobExecutor{
|
||||
db: database,
|
||||
logger: logger,
|
||||
cron: cron,
|
||||
jobs: jobsMap,
|
||||
jobMutex: jobMutex,
|
||||
transferExecutor: transferExec,
|
||||
notifier: notify,
|
||||
// metadataHandler: metadata,
|
||||
}
|
||||
}
|
||||
|
||||
// executeJob orchestrates the execution of a job by processing its configurations.
|
||||
func (je *JobExecutor) executeJob(jobID uint) {
|
||||
je.logger.LogDebug("Entering executeJob for job ID %d", jobID)
|
||||
defer je.logger.LogDebug("Exiting executeJob for job ID %d", jobID)
|
||||
|
||||
je.logger.LogInfo("Starting execution of job %d", jobID)
|
||||
|
||||
// Get job details
|
||||
var job db.Job
|
||||
if err := je.db.First(&job, jobID).Error; err != nil {
|
||||
je.logger.LogError("Error loading job %d: %v", jobID, err)
|
||||
return
|
||||
}
|
||||
|
||||
je.logger.LogDebug("Loaded job details: %+v", job)
|
||||
|
||||
// Get all configurations associated with this job
|
||||
configs, err := je.db.GetConfigsForJob(jobID)
|
||||
if err != nil {
|
||||
je.logger.LogError("Error loading configurations for job %d: %v", jobID, err)
|
||||
return
|
||||
}
|
||||
|
||||
je.logger.LogDebug("Loaded %d configurations for job %d", len(configs), jobID)
|
||||
|
||||
if len(configs) == 0 {
|
||||
je.logger.LogError("Error: job %d has no associated configurations", jobID)
|
||||
return
|
||||
}
|
||||
|
||||
// Get the ordered config IDs from the job
|
||||
orderedConfigIDs := job.GetConfigIDsList()
|
||||
je.logger.LogDebug("Ordered config IDs for job %d: %v", jobID, orderedConfigIDs)
|
||||
|
||||
// Create a map of configs for easy lookup
|
||||
configMap := make(map[uint]db.TransferConfig)
|
||||
for _, config := range configs {
|
||||
configMap[config.ID] = config
|
||||
}
|
||||
|
||||
// Process configurations in the specified order
|
||||
var orderedConfigs []db.TransferConfig
|
||||
|
||||
// First, add configs in the order specified in the job's ConfigIDs
|
||||
for _, configID := range orderedConfigIDs {
|
||||
if config, exists := configMap[configID]; exists {
|
||||
orderedConfigs = append(orderedConfigs, config)
|
||||
delete(configMap, configID) // Remove from map to avoid duplicates
|
||||
}
|
||||
}
|
||||
|
||||
// Add any remaining configs not in the ordered list (shouldn't happen, but just in case)
|
||||
for _, config := range configMap {
|
||||
orderedConfigs = append(orderedConfigs, config)
|
||||
}
|
||||
|
||||
je.logger.LogInfo("Processing job %d with %d configurations in specified order", jobID, len(orderedConfigs))
|
||||
|
||||
// Log the order of execution
|
||||
for i, config := range orderedConfigs {
|
||||
je.logger.LogDebug("Execution order %d/%d: Config ID %d (%s)", i+1, len(orderedConfigs), config.ID, config.Name)
|
||||
}
|
||||
|
||||
// Update job last run time
|
||||
startTime := time.Now()
|
||||
job.LastRun = &startTime
|
||||
if err := je.db.UpdateJobStatus(&job); err != nil {
|
||||
je.logger.LogError("Error updating job last run time for job %d: %v", jobID, err)
|
||||
}
|
||||
|
||||
// Process each configuration in the specified order
|
||||
for i, config := range orderedConfigs {
|
||||
je.processConfiguration(&job, &config, i+1, len(orderedConfigs))
|
||||
}
|
||||
|
||||
// Update next run time after execution
|
||||
// Need access to the shared jobs map and mutex from Scheduler
|
||||
je.jobMutex.Lock()
|
||||
entryID, exists := je.jobs[jobID]
|
||||
je.jobMutex.Unlock()
|
||||
|
||||
if exists {
|
||||
entry := je.cron.Entry(entryID)
|
||||
nextRun := entry.Next
|
||||
job.NextRun = &nextRun
|
||||
je.logger.LogInfo("Next run time for job %d: %v", jobID, nextRun)
|
||||
if err := je.db.UpdateJobStatus(&job); err != nil {
|
||||
je.logger.LogError("Error updating job next run time for job %d: %v", jobID, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processConfiguration processes a single configuration step within a job.
|
||||
func (je *JobExecutor) processConfiguration(job *db.Job, config *db.TransferConfig, index int, totalConfigs int) {
|
||||
je.logger.LogDebug("Processing configuration %d: %+v", config.ID, config)
|
||||
|
||||
je.logger.LogInfo("Processing configuration %d (%d/%d) for job %d: source=%s:%s, dest=%s:%s",
|
||||
config.ID,
|
||||
index,
|
||||
totalConfigs,
|
||||
job.ID,
|
||||
config.SourceType,
|
||||
config.SourcePath,
|
||||
config.DestinationType,
|
||||
config.DestinationPath,
|
||||
)
|
||||
|
||||
// Create job history entry for this configuration
|
||||
history := &db.JobHistory{
|
||||
JobID: job.ID,
|
||||
ConfigID: config.ID,
|
||||
StartTime: time.Now(),
|
||||
Status: "running",
|
||||
FilesTransferred: 0,
|
||||
BytesTransferred: 0,
|
||||
ErrorMessage: "",
|
||||
}
|
||||
if err := je.db.CreateJobHistory(history); err != nil {
|
||||
je.logger.LogError("Error creating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
return
|
||||
}
|
||||
|
||||
je.logger.LogDebug("Creating job history record: %+v", history)
|
||||
|
||||
// Send webhook notification for job start
|
||||
// TODO: Ensure Notifier struct and its methods are correctly defined and initialized
|
||||
je.notifier.SendNotifications(job, history, config) // Assuming sendWebhookNotification is a method on Notifier
|
||||
|
||||
// Execute the configuration transfer
|
||||
// TODO: Ensure TransferExecutor struct and its methods are correctly defined and initialized
|
||||
je.transferExecutor.executeConfigTransfer(*job, *config, history) // Assuming executeConfigTransfer is a method on TransferExecutor
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/natefinch/lumberjack.v2"
|
||||
)
|
||||
|
||||
// LogLevel represents the verbosity level of logging
|
||||
type LogLevel int
|
||||
|
||||
const (
|
||||
// LogLevelError only logs errors
|
||||
LogLevelError LogLevel = iota
|
||||
// LogLevelInfo logs info and errors
|
||||
LogLevelInfo
|
||||
// LogLevelDebug logs everything including debug messages
|
||||
LogLevelDebug
|
||||
)
|
||||
|
||||
// String returns the string representation of a log level
|
||||
func (l LogLevel) String() string {
|
||||
switch l {
|
||||
case LogLevelError:
|
||||
return "error"
|
||||
case LogLevelInfo:
|
||||
return "info"
|
||||
case LogLevelDebug:
|
||||
return "debug"
|
||||
default:
|
||||
return "unknown"
|
||||
}
|
||||
}
|
||||
|
||||
// ParseLogLevel parses a string into a LogLevel
|
||||
func ParseLogLevel(level string) LogLevel {
|
||||
switch strings.ToLower(level) {
|
||||
case "error":
|
||||
return LogLevelError
|
||||
case "info":
|
||||
return LogLevelInfo
|
||||
case "debug":
|
||||
return LogLevelDebug
|
||||
default:
|
||||
return LogLevelInfo // Default to info level
|
||||
}
|
||||
}
|
||||
|
||||
// Logger handles log output to file and console
|
||||
type Logger struct {
|
||||
Info *log.Logger
|
||||
Error *log.Logger
|
||||
Debug *log.Logger
|
||||
file *lumberjack.Logger
|
||||
logLevel LogLevel
|
||||
}
|
||||
|
||||
// LogInfo logs an info message if the log level allows it
|
||||
func (l *Logger) LogInfo(format string, v ...interface{}) {
|
||||
if l.logLevel >= LogLevelInfo {
|
||||
l.Info.Printf(format, v...)
|
||||
}
|
||||
}
|
||||
|
||||
// LogError logs an error message if the log level allows it
|
||||
func (l *Logger) LogError(format string, v ...interface{}) {
|
||||
if l.logLevel >= LogLevelError {
|
||||
l.Error.Printf(format, v...)
|
||||
}
|
||||
}
|
||||
|
||||
// LogDebug logs a debug message if the log level allows it
|
||||
func (l *Logger) LogDebug(format string, v ...interface{}) {
|
||||
if l.logLevel >= LogLevelDebug {
|
||||
l.Debug.Printf(format, v...)
|
||||
}
|
||||
}
|
||||
|
||||
// NewLogger creates a new logger that writes to both file and console
|
||||
func NewLogger() *Logger {
|
||||
// Get data directory from environment or use default
|
||||
dataDir := os.Getenv("DATA_DIR")
|
||||
if dataDir == "" {
|
||||
dataDir = "./data"
|
||||
}
|
||||
|
||||
// Ensure logs directory exists
|
||||
logsDir := filepath.Join(dataDir, "logs")
|
||||
if envLogsDir := os.Getenv("LOGS_DIR"); envLogsDir != "" {
|
||||
logsDir = envLogsDir
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(logsDir, 0755); err != nil {
|
||||
fmt.Printf("Error creating logs directory: %v\n", err)
|
||||
}
|
||||
|
||||
// Get log rotation settings from environment or use defaults
|
||||
maxSize := 10 // Default: 10MB
|
||||
if envSize := os.Getenv("LOG_MAX_SIZE"); envSize != "" {
|
||||
if size, err := strconv.Atoi(envSize); err == nil && size > 0 {
|
||||
maxSize = size
|
||||
}
|
||||
}
|
||||
|
||||
maxBackups := 5 // Default: keep 5 backups
|
||||
if envBackups := os.Getenv("LOG_MAX_BACKUPS"); envBackups != "" {
|
||||
if backups, err := strconv.Atoi(envBackups); err == nil && backups >= 0 {
|
||||
maxBackups = backups
|
||||
}
|
||||
}
|
||||
|
||||
maxAge := 30 // Default: 30 days
|
||||
if envAge := os.Getenv("LOG_MAX_AGE"); envAge != "" {
|
||||
if age, err := strconv.Atoi(envAge); err == nil && age >= 0 {
|
||||
maxAge = age
|
||||
}
|
||||
}
|
||||
|
||||
compress := true // Default: compress logs
|
||||
if envCompress := os.Getenv("LOG_COMPRESS"); envCompress == "false" {
|
||||
compress = false
|
||||
}
|
||||
|
||||
// Get log level from environment or use default
|
||||
logLevel := LogLevelInfo // Default to info level
|
||||
if envLogLevel := os.Getenv("LOG_LEVEL"); envLogLevel != "" {
|
||||
logLevel = ParseLogLevel(envLogLevel)
|
||||
}
|
||||
|
||||
// Setup log rotation
|
||||
logFile := &lumberjack.Logger{
|
||||
Filename: filepath.Join(logsDir, "scheduler.log"),
|
||||
MaxSize: maxSize,
|
||||
MaxBackups: maxBackups,
|
||||
MaxAge: maxAge,
|
||||
Compress: compress,
|
||||
}
|
||||
|
||||
// Create multi-writer for both file and console
|
||||
consoleAndFile := io.MultiWriter(os.Stdout, logFile)
|
||||
|
||||
// Create loggers with different prefixes
|
||||
logger := &Logger{
|
||||
Info: log.New(consoleAndFile, "INFO: ", log.Ldate|log.Ltime),
|
||||
Error: log.New(consoleAndFile, "ERROR: ", log.Ldate|log.Ltime),
|
||||
Debug: log.New(consoleAndFile, "DEBUG: ", log.Ldate|log.Ltime),
|
||||
file: logFile,
|
||||
logLevel: logLevel,
|
||||
}
|
||||
|
||||
// Log rotation settings and log level
|
||||
if logLevel >= LogLevelInfo {
|
||||
logger.Info.Printf("Log rotation configured: file=%s, maxSize=%dMB, maxBackups=%d, maxAge=%d days, compress=%v, logLevel=%s",
|
||||
filepath.Join(logsDir, "scheduler.log"), maxSize, maxBackups, maxAge, compress, logLevel.String())
|
||||
}
|
||||
|
||||
if logLevel >= LogLevelDebug {
|
||||
logger.Debug.Printf("Log rotation details: file=%s, maxSize=%dMB, maxBackups=%d, maxAge=%d days, compress=%v",
|
||||
filepath.Join(logsDir, "scheduler.log"), maxSize, maxBackups, maxAge, compress)
|
||||
}
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// Close closes the log file
|
||||
func (l *Logger) Close() {
|
||||
if l.file != nil {
|
||||
l.file.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// RotateLogs manually triggers log rotation
|
||||
func (l *Logger) RotateLogs() error {
|
||||
if l.file != nil {
|
||||
return l.file.Rotate()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/starfleetcptn/gomft/internal/db"
|
||||
)
|
||||
|
||||
// MetadataHandler handles checking file processing history.
|
||||
type MetadataHandler struct {
|
||||
db *db.DB
|
||||
logger *Logger // Added logger dependency
|
||||
}
|
||||
|
||||
// NewMetadataHandler creates a new MetadataHandler.
|
||||
func NewMetadataHandler(database *db.DB, logger *Logger) *MetadataHandler {
|
||||
return &MetadataHandler{
|
||||
db: database,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
// hasFileBeenProcessed checks if a file with the same hash has been processed before.
|
||||
func (mh *MetadataHandler) hasFileBeenProcessed(jobID uint, fileHash string) (bool, *db.FileMetadata, error) {
|
||||
if fileHash == "" {
|
||||
return false, nil, nil
|
||||
}
|
||||
|
||||
// First try to find by hash (most reliable)
|
||||
metadata, err := mh.db.GetFileMetadataByHash(fileHash)
|
||||
if err == nil && metadata != nil {
|
||||
// Optional: Add logging here if needed
|
||||
mh.logger.LogDebug("Found existing metadata by hash for job %d, hash %s", jobID, fileHash)
|
||||
return true, metadata, nil
|
||||
}
|
||||
if err != nil {
|
||||
mh.logger.LogError("Error checking metadata by hash for job %d, hash %s: %v", jobID, fileHash, err)
|
||||
}
|
||||
|
||||
return false, nil, err // Return the error if one occurred during DB lookup
|
||||
}
|
||||
|
||||
// checkFileProcessingHistory checks processing history for a given file name within a specific job.
|
||||
func (mh *MetadataHandler) checkFileProcessingHistory(jobID uint, fileName string) (*db.FileMetadata, error) {
|
||||
// Try to find by job and filename
|
||||
metadata, err := mh.db.GetFileMetadataByJobAndName(jobID, fileName)
|
||||
if err == nil && metadata != nil {
|
||||
mh.logger.LogDebug("Found existing metadata by name for job %d, file %s", jobID, fileName)
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
mh.logger.LogError("Error checking metadata by name for job %d, file %s: %v", jobID, fileName, err)
|
||||
// Don't return error here, just indicate not found
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("no history found for file %s in job %d", fileName, jobID)
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+42
-2467
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,927 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/starfleetcptn/gomft/internal/db"
|
||||
)
|
||||
|
||||
// TransferExecutor handles the rclone command execution and transfer logic.
|
||||
type TransferExecutor struct {
|
||||
db *db.DB
|
||||
logger *Logger
|
||||
metadataHandler *MetadataHandler // Placeholder
|
||||
notifier *Notifier // Placeholder
|
||||
}
|
||||
|
||||
// NewTransferExecutor creates a new TransferExecutor.
|
||||
func NewTransferExecutor(
|
||||
database *db.DB,
|
||||
logger *Logger,
|
||||
metadata *MetadataHandler,
|
||||
notify *Notifier,
|
||||
) *TransferExecutor {
|
||||
return &TransferExecutor{
|
||||
db: database,
|
||||
logger: logger,
|
||||
metadataHandler: metadata,
|
||||
notifier: notify,
|
||||
}
|
||||
}
|
||||
|
||||
// executeConfigTransfer performs the actual file transfer for a single configuration
|
||||
func (te *TransferExecutor) executeConfigTransfer(job db.Job, config db.TransferConfig, history *db.JobHistory) {
|
||||
te.logger.LogDebug("Starting transfer for config %d with params: %+v", config.ID, config)
|
||||
|
||||
// Track files already processed in this job execution to prevent duplicates
|
||||
processedFiles := make(map[string]bool)
|
||||
|
||||
// Get rclone config path
|
||||
configPath := te.db.GetConfigRclonePath(&config)
|
||||
|
||||
// Get the command to use for the transfer
|
||||
var rcloneCommand string = "copyto" // Default command
|
||||
if config.CommandID > 0 {
|
||||
// Get the command by ID
|
||||
command, err := te.db.GetRcloneCommand(config.CommandID)
|
||||
if err == nil && command != nil {
|
||||
rcloneCommand = command.Name
|
||||
te.logger.LogDebug("Using rclone command %s for job %d, config %d", rcloneCommand, job.ID, config.ID)
|
||||
} else {
|
||||
te.logger.LogError("Failed to get rclone command with ID %d: %v", config.CommandID, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Determine command type to handle execution appropriately
|
||||
commandType := determineCommandType(rcloneCommand) // Package-level call
|
||||
te.logger.LogDebug("Command %s is of type: %s", rcloneCommand, commandType)
|
||||
|
||||
// For non-file-by-file transfer commands, use the simple execution approach
|
||||
if commandType != "transfer" || isDirectoryBasedTransfer(rcloneCommand) { // Package-level call
|
||||
te.executeSimpleCommand(rcloneCommand, commandType, job, config, history, configPath)
|
||||
return
|
||||
}
|
||||
|
||||
// The rest of the function handles file-by-file transfer commands (copyto, moveto)
|
||||
// Use lsjson to get file list and metadata in one operation instead of separate size and ls commands
|
||||
listArgs := []string{
|
||||
"--config", configPath,
|
||||
"lsjson",
|
||||
"--hash",
|
||||
"--recursive",
|
||||
}
|
||||
|
||||
// Add file pattern filter if specified
|
||||
if config.FilePattern != "" && config.FilePattern != "*" {
|
||||
// Create a temporary filter file for complex patterns
|
||||
filterFile, err := createRcloneFilterFile(config.FilePattern) // Package-level call from utils.go
|
||||
if err != nil {
|
||||
te.logger.LogError("Error creating filter file for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
history.Status = "failed"
|
||||
history.ErrorMessage = fmt.Sprintf("Filter Creation Error: %v", err)
|
||||
endTime := time.Now()
|
||||
history.EndTime = &endTime
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
// Send notification for failure
|
||||
te.notifier.SendNotifications(&job, history, &config) // Call via notifier
|
||||
|
||||
return
|
||||
}
|
||||
defer os.Remove(filterFile)
|
||||
listArgs = append(listArgs, "--filter-from", filterFile)
|
||||
}
|
||||
|
||||
// Add source path with bucket for S3-compatible storage
|
||||
var sourceListPath string
|
||||
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
|
||||
sourceListPath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
|
||||
if config.SourcePath != "" && config.SourcePath != "/" {
|
||||
sourceListPath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
|
||||
}
|
||||
} else {
|
||||
sourceListPath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
|
||||
}
|
||||
|
||||
listArgs = append(listArgs, sourceListPath)
|
||||
|
||||
// Execute lsjson command
|
||||
te.logger.LogDebug("Full lsjson command: %s %v", os.Getenv("RCLONE_PATH"), listArgs)
|
||||
rclonePath := os.Getenv("RCLONE_PATH")
|
||||
if rclonePath == "" {
|
||||
rclonePath = "rclone"
|
||||
}
|
||||
listCmd := exec.Command(rclonePath, listArgs...)
|
||||
listOutput, listErr := listCmd.CombinedOutput()
|
||||
|
||||
// Add debug logging of raw output
|
||||
if listErr == nil {
|
||||
te.logger.LogDebug("Raw lsjson output for job %d config %d:\n%s",
|
||||
job.ID,
|
||||
config.ID,
|
||||
string(listOutput))
|
||||
} else {
|
||||
te.logger.LogDebug("Raw lsjson output (error case) for job %d config %d:\n%s",
|
||||
job.ID,
|
||||
config.ID,
|
||||
string(listOutput))
|
||||
}
|
||||
|
||||
if listErr != nil {
|
||||
te.logger.LogError("Error listing files for job %d, config %d: %v", job.ID, config.ID, listErr)
|
||||
history.Status = "failed"
|
||||
history.ErrorMessage = fmt.Sprintf("File Listing Error: %v\nOutput: %s", listErr, string(listOutput))
|
||||
endTime := time.Now()
|
||||
history.EndTime = &endTime
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
// Send notification for failure
|
||||
te.notifier.SendNotifications(&job, history, &config) // Call via notifier
|
||||
return
|
||||
}
|
||||
|
||||
// Parse JSON output to get file information
|
||||
var fileEntries []map[string]interface{}
|
||||
if err := json.Unmarshal(listOutput, &fileEntries); err != nil {
|
||||
te.logger.LogError("Error parsing file list JSON for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
history.Status = "failed"
|
||||
history.ErrorMessage = fmt.Sprintf("JSON Parsing Error: %v", err)
|
||||
endTime := time.Now()
|
||||
history.EndTime = &endTime
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
// Send notification for failure
|
||||
te.notifier.SendNotifications(&job, history, &config) // Call via notifier
|
||||
return
|
||||
}
|
||||
|
||||
// Calculate total size and filter out directories
|
||||
var files []map[string]interface{}
|
||||
var totalSize int64
|
||||
for _, entry := range fileEntries {
|
||||
// Process directories
|
||||
if isDir, ok := entry["IsDir"].(bool); ok && isDir {
|
||||
continue
|
||||
}
|
||||
|
||||
// Add to files list
|
||||
files = append(files, entry)
|
||||
|
||||
// Add to total size
|
||||
if size, ok := entry["Size"].(float64); ok {
|
||||
totalSize += int64(size)
|
||||
}
|
||||
}
|
||||
|
||||
te.logger.LogInfo("Found %d files totaling %d bytes to transfer for job %d, config %d", len(files), totalSize, job.ID, config.ID)
|
||||
|
||||
// Update history with size information
|
||||
history.BytesTransferred = totalSize
|
||||
|
||||
if len(files) == 0 {
|
||||
te.logger.LogInfo("No files to transfer for job %d, config %d", job.ID, config.ID)
|
||||
history.Status = "completed"
|
||||
history.ErrorMessage = ""
|
||||
history.FilesTransferred = 0
|
||||
endTime := time.Now()
|
||||
history.EndTime = &endTime
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
// Send notification for empty completion
|
||||
te.notifier.SendNotifications(&job, history, &config) // Call via notifier
|
||||
return
|
||||
}
|
||||
|
||||
var transferErrors []string
|
||||
filesTransferred := 0
|
||||
|
||||
// Use mutex for thread-safe access to shared variables
|
||||
var mutex sync.Mutex
|
||||
|
||||
// Determine number of concurrent transfers
|
||||
maxConcurrent := config.MaxConcurrentTransfers
|
||||
if maxConcurrent < 1 {
|
||||
maxConcurrent = 1 // Default to 1 if not set
|
||||
}
|
||||
|
||||
// Limit Google Photos to 1 concurrent transfers
|
||||
if config.SourceType == "gphotos" || config.DestinationType == "gphotos" {
|
||||
maxConcurrent = 1
|
||||
}
|
||||
|
||||
te.logger.LogInfo("Using %d concurrent transfers for job %d, config %d", maxConcurrent, job.ID, config.ID)
|
||||
|
||||
// Create wait group for concurrent processing
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Create channel to limit concurrency
|
||||
concurrencySemaphore := make(chan struct{}, maxConcurrent)
|
||||
|
||||
// Process each file individually
|
||||
for i, fileEntry := range files {
|
||||
fileName, ok := fileEntry["Path"].(string)
|
||||
if !ok || fileName == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip files that have already been processed in this execution
|
||||
if processedFiles[fileName] {
|
||||
te.logger.LogDebug("Skipping duplicate file entry: %s (already processed in this execution)", fileName)
|
||||
continue
|
||||
}
|
||||
|
||||
// Extract hash from the file entry
|
||||
fileHash := ""
|
||||
if hashes, ok := fileEntry["Hashes"].(map[string]interface{}); ok {
|
||||
// Try several hash algorithms in order of preference
|
||||
for _, hashType := range []string{"SHA-1", "sha1", "MD5", "md5", "sha256", "crc32"} {
|
||||
if hashValue, found := hashes[hashType]; found {
|
||||
if hashStr, ok := hashValue.(string); ok && hashStr != "" {
|
||||
te.logger.LogDebug("Found hash %s: %s for file %s", hashType, hashStr, fileName)
|
||||
fileHash = hashStr
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Log if no hash was found
|
||||
if fileHash == "" {
|
||||
te.logger.LogDebug("No hash found for file %s. Available fields: %v", fileName, fileEntry)
|
||||
}
|
||||
|
||||
// Extract size from the file entry
|
||||
fileSize := int64(0)
|
||||
if size, ok := fileEntry["Size"].(float64); ok {
|
||||
fileSize = int64(size)
|
||||
}
|
||||
|
||||
// Skip files that have already been processed based on hash
|
||||
skipFiles := config.GetSkipProcessedFiles()
|
||||
|
||||
if skipFiles && fileHash != "" {
|
||||
// Call via metadataHandler
|
||||
alreadyProcessed, prevMetadata, err := te.metadataHandler.hasFileBeenProcessed(job.ID, fileHash)
|
||||
if err == nil && alreadyProcessed {
|
||||
te.logger.LogDebug("File %s with hash %s was previously processed on %s with status: %s",
|
||||
fileName, fileHash, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
|
||||
|
||||
// Determine if we should skip this file based on status
|
||||
shouldSkip := false
|
||||
if prevMetadata.Status == "processed" ||
|
||||
prevMetadata.Status == "archived" ||
|
||||
prevMetadata.Status == "deleted" ||
|
||||
prevMetadata.Status == "archived_and_deleted" {
|
||||
shouldSkip = true
|
||||
}
|
||||
|
||||
if shouldSkip {
|
||||
te.logger.LogInfo("Skipping unchanged file %s (hash matches previous processing)", fileName)
|
||||
continue
|
||||
} else {
|
||||
te.logger.LogInfo("Re-processing file %s despite previous processing (skipProcessedFiles=%v)", fileName, skipFiles)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Also check the processing history for this specific file name
|
||||
// Call via metadataHandler
|
||||
prevMetadata, histErr := te.metadataHandler.checkFileProcessingHistory(job.ID, fileName)
|
||||
if histErr == nil {
|
||||
te.logger.LogDebug("File %s was previously processed on %s with status: %s",
|
||||
fileName, prevMetadata.ProcessedTime.Format(time.RFC3339), prevMetadata.Status)
|
||||
|
||||
// Determine if we should skip this file based on name+hash match
|
||||
shouldSkip := false
|
||||
if skipFiles && fileHash != "" && fileHash == prevMetadata.FileHash {
|
||||
if prevMetadata.Status == "processed" ||
|
||||
prevMetadata.Status == "archived" ||
|
||||
prevMetadata.Status == "deleted" ||
|
||||
prevMetadata.Status == "archived_and_deleted" {
|
||||
shouldSkip = true
|
||||
}
|
||||
}
|
||||
|
||||
if shouldSkip {
|
||||
te.logger.LogInfo("Skipping unchanged file %s (hash matches previous processing)", fileName)
|
||||
// Skip this file and continue to the next one
|
||||
continue
|
||||
} else if fileHash != "" && fileHash == prevMetadata.FileHash {
|
||||
te.logger.LogInfo("Re-processing file %s despite matching hash (skipProcessedFiles=%v)", fileName, skipFiles)
|
||||
}
|
||||
}
|
||||
|
||||
// Mark this file as processed for this execution before launching goroutine
|
||||
// to prevent duplicate processing
|
||||
processedFiles[fileName] = true
|
||||
|
||||
// Add to wait group before starting goroutine
|
||||
wg.Add(1)
|
||||
|
||||
// Get creation time and mod time for the file metadata
|
||||
createTime := time.Now()
|
||||
modTime := time.Now()
|
||||
if creationTimeStr, ok := fileEntry["ModTime"].(string); ok {
|
||||
if t, err := time.Parse(time.RFC3339Nano, creationTimeStr); err == nil {
|
||||
modTime = t
|
||||
createTime = t
|
||||
}
|
||||
}
|
||||
|
||||
// Capture current file information for goroutine
|
||||
currentFileName := fileName
|
||||
currentFileHash := fileHash
|
||||
currentFileSize := fileSize
|
||||
currentCreateTime := createTime
|
||||
currentModTime := modTime
|
||||
|
||||
// Log the file information that will be processed
|
||||
te.logger.LogDebug("Processing file %d/%d: %s (Size: %d, Hash: %s)",
|
||||
i+1, len(files), currentFileName, currentFileSize, currentFileHash)
|
||||
|
||||
// Start goroutine for concurrent processing
|
||||
go func() {
|
||||
// Acquire semaphore
|
||||
concurrencySemaphore <- struct{}{}
|
||||
defer func() {
|
||||
// Release semaphore and mark work as done
|
||||
<-concurrencySemaphore
|
||||
wg.Done()
|
||||
}()
|
||||
|
||||
// Prepare rclone command
|
||||
transferArgs := te.prepareBaseArguments(rcloneCommand, &config, nil) // Use method call
|
||||
|
||||
// Source and destination paths
|
||||
var sourcePath, destPath string
|
||||
|
||||
// For S3, MinIO, and B2, include the bucket in the path
|
||||
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, currentFileName)
|
||||
if config.SourcePath != "" && config.SourcePath != "/" {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s/%s/%s", config.ID, config.SourceBucket, config.SourcePath, currentFileName)
|
||||
}
|
||||
} else {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourcePath, currentFileName)
|
||||
}
|
||||
|
||||
var destFile string = currentFileName
|
||||
|
||||
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, currentFileName)
|
||||
if config.DestinationPath != "" && config.DestinationPath != "/" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", config.ID, config.DestBucket, config.DestinationPath, currentFileName)
|
||||
}
|
||||
} else {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestinationPath, currentFileName)
|
||||
}
|
||||
|
||||
// Add output filename pattern if specified
|
||||
if config.OutputPattern != "" {
|
||||
// Process the output pattern for this specific file
|
||||
destFile = ProcessOutputPattern(config.OutputPattern, currentFileName) // Package-level call from utils.go
|
||||
|
||||
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, destFile)
|
||||
if config.DestinationPath != "" && config.DestinationPath != "/" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s/%s", config.ID, config.DestBucket, config.DestinationPath, destFile)
|
||||
}
|
||||
} else {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestinationPath, destFile)
|
||||
}
|
||||
|
||||
te.logger.LogDebug("Renaming file from %s to %s for job %d, config %d", currentFileName, destFile, job.ID, config.ID)
|
||||
}
|
||||
|
||||
// Add source and destination to the command (already added in prepareBaseArguments for some commands, check logic)
|
||||
// This part needs careful review based on how prepareBaseArguments is structured
|
||||
// For file-by-file (copyto, moveto), we need source and dest here.
|
||||
transferArgs = append(transferArgs, sourcePath, destPath)
|
||||
|
||||
// Execute transfer for this file
|
||||
te.logger.LogDebug("Full transfer command: %s %v", rclonePath, transferArgs)
|
||||
te.logger.LogDebug("Environment: RCLONE_PATH=%s", os.Getenv("RCLONE_PATH"))
|
||||
cmd := exec.Command(rclonePath, transferArgs...)
|
||||
fileOutput, fileErr := cmd.CombinedOutput()
|
||||
|
||||
// Print the output
|
||||
te.logger.LogDebug("Output for file %s: %s", currentFileName, string(fileOutput))
|
||||
|
||||
// Create file metadata record
|
||||
fileStatus := "processed"
|
||||
var fileErrorMsg string
|
||||
var destPathForDB string
|
||||
|
||||
// Check if file was successfully transferred
|
||||
if fileErr != nil {
|
||||
te.logger.LogError("Error transferring file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, fileErr)
|
||||
mutex.Lock()
|
||||
transferErrors = append(transferErrors, fmt.Sprintf("File %s: %v", currentFileName, fileErr))
|
||||
mutex.Unlock()
|
||||
fileStatus = "error"
|
||||
fileErrorMsg = fileErr.Error()
|
||||
} else {
|
||||
mutex.Lock()
|
||||
filesTransferred++
|
||||
mutex.Unlock()
|
||||
te.logger.LogInfo("Successfully transferred file %s for job %d, config %d", currentFileName, job.ID, config.ID)
|
||||
|
||||
// Extract the actual destination path (without rclone remote prefix)
|
||||
if config.DestinationType == "local" {
|
||||
destPathForDB = filepath.Join(config.DestinationPath, destFile)
|
||||
} else {
|
||||
// For remote destinations, store the path format
|
||||
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
|
||||
if config.DestinationPath != "" && config.DestinationPath != "/" {
|
||||
destPathForDB = fmt.Sprintf("%s/%s/%s", config.DestBucket, config.DestinationPath, destFile)
|
||||
} else {
|
||||
destPathForDB = fmt.Sprintf("%s/%s", config.DestBucket, destFile)
|
||||
}
|
||||
} else {
|
||||
destPathForDB = fmt.Sprintf("%s/%s", config.DestinationPath, destFile)
|
||||
}
|
||||
}
|
||||
|
||||
// If archiving is enabled and transfer was successful, move files to archive
|
||||
if config.GetArchiveEnabled() && config.ArchivePath != "" {
|
||||
te.logger.LogInfo("Archiving file %s for job %d, config %d", currentFileName, job.ID, config.ID)
|
||||
|
||||
// We don't need to move the file since we used moveto, but we can copy it to archive
|
||||
archiveArgs := []string{
|
||||
"--config", configPath,
|
||||
"copyto",
|
||||
sourcePath,
|
||||
}
|
||||
|
||||
// Construct archive path with bucket if needed
|
||||
var archiveDest string
|
||||
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
|
||||
archiveDest = fmt.Sprintf("source_%d:%s/%s/%s", config.ID, config.SourceBucket, config.ArchivePath, currentFileName)
|
||||
} else {
|
||||
archiveDest = fmt.Sprintf("source_%d:%s/%s", config.ID, config.ArchivePath, currentFileName)
|
||||
}
|
||||
|
||||
archiveArgs = append(archiveArgs, archiveDest)
|
||||
|
||||
te.logger.LogInfo("Executing rclone archive command for job %d, config %d, file %s: rclone %s",
|
||||
job.ID, config.ID, currentFileName, strings.Join(archiveArgs, " "))
|
||||
// Get the rclone path from the environment variable or use the default path
|
||||
rclonePath := os.Getenv("RCLONE_PATH")
|
||||
if rclonePath == "" {
|
||||
rclonePath = "rclone"
|
||||
}
|
||||
archiveCmd := exec.Command(rclonePath, archiveArgs...)
|
||||
archiveOutput, archiveErr := archiveCmd.CombinedOutput()
|
||||
|
||||
// Print the output
|
||||
te.logger.LogDebug("Output for file %s: %s", currentFileName, string(archiveOutput))
|
||||
|
||||
// Check if file was successfully transferred
|
||||
if archiveErr != nil {
|
||||
te.logger.LogError("Warning: Error archiving file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, archiveErr)
|
||||
mutex.Lock()
|
||||
transferErrors = append(transferErrors,
|
||||
fmt.Sprintf("Archive error for file %s: %v", currentFileName, archiveErr))
|
||||
mutex.Unlock()
|
||||
} else {
|
||||
fileStatus = "archived"
|
||||
}
|
||||
}
|
||||
|
||||
if config.GetDeleteAfterTransfer() {
|
||||
te.logger.LogInfo("Deleting file %s for job %d, config %d", currentFileName, job.ID, config.ID)
|
||||
deleteArgs := []string{
|
||||
"--config", configPath,
|
||||
"deletefile",
|
||||
sourcePath}
|
||||
deleteCmd := exec.Command(rclonePath, deleteArgs...)
|
||||
deleteOutput, deleteErr := deleteCmd.CombinedOutput()
|
||||
te.logger.LogDebug("Output for file %s: %s", currentFileName, string(deleteOutput))
|
||||
if deleteErr != nil {
|
||||
te.logger.LogError("Error deleting file %s for job %d, config %d: %v", currentFileName, job.ID, config.ID, deleteErr)
|
||||
mutex.Lock()
|
||||
transferErrors = append(transferErrors,
|
||||
fmt.Sprintf("Delete error for file %s: %v", currentFileName, deleteErr))
|
||||
mutex.Unlock()
|
||||
} else {
|
||||
if fileStatus == "archived" {
|
||||
fileStatus = "archived_and_deleted"
|
||||
} else {
|
||||
fileStatus = "deleted"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Create and save file metadata
|
||||
metadata := &db.FileMetadata{
|
||||
JobID: job.ID,
|
||||
ConfigID: config.ID,
|
||||
FileName: currentFileName,
|
||||
OriginalPath: config.SourcePath,
|
||||
FileSize: currentFileSize,
|
||||
FileHash: currentFileHash,
|
||||
CreationTime: currentCreateTime,
|
||||
ModTime: currentModTime,
|
||||
ProcessedTime: time.Now(),
|
||||
DestinationPath: destPathForDB,
|
||||
Status: fileStatus,
|
||||
ErrorMessage: fileErrorMsg,
|
||||
}
|
||||
|
||||
if err := te.db.CreateFileMetadata(metadata); err != nil {
|
||||
te.logger.LogError("Error creating file metadata for %s: %v", currentFileName, err)
|
||||
} else {
|
||||
te.logger.LogDebug("Created file metadata record for %s (ID: %d) with hash: %s", currentFileName, metadata.ID, currentFileHash)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Wait for all transfers to complete
|
||||
wg.Wait()
|
||||
|
||||
// Clean up concurrency semaphore
|
||||
close(concurrencySemaphore)
|
||||
|
||||
// Update job history with transfer results
|
||||
history.FilesTransferred = filesTransferred
|
||||
|
||||
if len(transferErrors) > 0 {
|
||||
history.Status = "completed_with_errors"
|
||||
history.ErrorMessage = fmt.Sprintf("Transfer completed with %d errors:\n%s",
|
||||
len(transferErrors), strings.Join(transferErrors, "\n"))
|
||||
} else {
|
||||
history.Status = "completed"
|
||||
}
|
||||
|
||||
// Update job history with completion status and end time
|
||||
endTime := time.Now()
|
||||
history.EndTime = &endTime
|
||||
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
|
||||
// Create job notification
|
||||
if err := te.notifier.createJobNotification(&job, history); err != nil {
|
||||
te.logger.LogError("Failed to create job notification", "jobID", job.ID, "error", err)
|
||||
}
|
||||
|
||||
// Send notification for success or with errors
|
||||
te.notifier.SendNotifications(&job, history, &config)
|
||||
}
|
||||
|
||||
// isDirectoryBasedTransfer checks if a transfer command operates on directories rather than individual files
|
||||
func isDirectoryBasedTransfer(commandName string) bool {
|
||||
// These commands operate on entire directories, not file-by-file
|
||||
dirBasedCommands := map[string]bool{
|
||||
"sync": true,
|
||||
"bisync": true,
|
||||
"copy": true,
|
||||
"move": true,
|
||||
}
|
||||
|
||||
return dirBasedCommands[commandName]
|
||||
}
|
||||
|
||||
// determineCommandType categorizes rclone commands into types for execution
|
||||
func determineCommandType(commandName string) string {
|
||||
// File transfer commands
|
||||
transferCommands := map[string]bool{
|
||||
"copy": true,
|
||||
"copyto": true,
|
||||
"move": true,
|
||||
"moveto": true,
|
||||
"sync": true,
|
||||
"bisync": true,
|
||||
}
|
||||
|
||||
// Listing commands
|
||||
listingCommands := map[string]bool{
|
||||
"ls": true,
|
||||
"lsd": true,
|
||||
"lsl": true,
|
||||
"lsf": true,
|
||||
"lsjson": true,
|
||||
"listremotes": true,
|
||||
}
|
||||
|
||||
// Information commands
|
||||
infoCommands := map[string]bool{
|
||||
"md5sum": true,
|
||||
"sha1sum": true,
|
||||
"size": true,
|
||||
"version": true,
|
||||
}
|
||||
|
||||
// Directory operations
|
||||
dirCommands := map[string]bool{
|
||||
"mkdir": true,
|
||||
"rmdir": true,
|
||||
"rmdirs": true,
|
||||
}
|
||||
|
||||
// Destructive commands
|
||||
destructiveCommands := map[string]bool{
|
||||
"delete": true,
|
||||
"purge": true,
|
||||
}
|
||||
|
||||
// Maintenance commands
|
||||
maintenanceCommands := map[string]bool{
|
||||
"cleanup": true,
|
||||
"dedupe": true,
|
||||
"check": true,
|
||||
}
|
||||
|
||||
// Specialized commands
|
||||
specialCommands := map[string]bool{
|
||||
"obscure": true,
|
||||
"cryptcheck": true,
|
||||
}
|
||||
|
||||
// Determine the command type
|
||||
if transferCommands[commandName] {
|
||||
return "transfer"
|
||||
} else if listingCommands[commandName] {
|
||||
return "listing"
|
||||
} else if infoCommands[commandName] {
|
||||
return "info"
|
||||
} else if dirCommands[commandName] {
|
||||
return "directory"
|
||||
} else if destructiveCommands[commandName] {
|
||||
return "destructive"
|
||||
} else if maintenanceCommands[commandName] {
|
||||
return "maintenance"
|
||||
} else if specialCommands[commandName] {
|
||||
return "special"
|
||||
}
|
||||
|
||||
// Default to transfer if unknown
|
||||
return "transfer"
|
||||
}
|
||||
|
||||
// executeSimpleCommand executes a simple command (non file-by-file transfer)
|
||||
func (te *TransferExecutor) executeSimpleCommand(cmdName string, cmdType string, job db.Job, config db.TransferConfig, history *db.JobHistory, configPath string) {
|
||||
te.logger.LogInfo("Executing simple command '%s' of type '%s' for job %d, config %d", cmdName, cmdType, job.ID, config.ID)
|
||||
|
||||
// Prepare base arguments
|
||||
baseArgs := te.prepareBaseArguments(cmdName, &config, nil) // Use method call
|
||||
|
||||
// Prepare source and destination paths
|
||||
var sourcePath, destPath string
|
||||
|
||||
// Handle source path with bucket for S3-compatible storage
|
||||
if config.SourceType == "s3" || config.SourceType == "minio" || config.SourceType == "b2" {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourceBucket)
|
||||
if config.SourcePath != "" && config.SourcePath != "/" {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s/%s", config.ID, config.SourceBucket, config.SourcePath)
|
||||
}
|
||||
} else {
|
||||
sourcePath = fmt.Sprintf("source_%d:%s", config.ID, config.SourcePath)
|
||||
}
|
||||
|
||||
// Handle destination path with bucket for S3-compatible storage
|
||||
if config.DestinationType == "s3" || config.DestinationType == "minio" || config.DestinationType == "b2" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestBucket)
|
||||
if config.DestinationPath != "" && config.DestinationPath != "/" {
|
||||
destPath = fmt.Sprintf("dest_%d:%s/%s", config.ID, config.DestBucket, config.DestinationPath)
|
||||
}
|
||||
} else {
|
||||
destPath = fmt.Sprintf("dest_%d:%s", config.ID, config.DestinationPath)
|
||||
}
|
||||
|
||||
// Add appropriate paths based on command type
|
||||
args := baseArgs // Start with base args prepared by prepareBaseArguments
|
||||
switch cmdType {
|
||||
case "transfer":
|
||||
// Directory-based transfers and file-specific transfers handled here
|
||||
args = append(args, sourcePath, destPath)
|
||||
case "maintenance":
|
||||
// Check command needs both source and destination, others may just need source
|
||||
if cmdName == "check" {
|
||||
args = append(args, sourcePath, destPath)
|
||||
} else {
|
||||
args = append(args, sourcePath)
|
||||
}
|
||||
case "listing":
|
||||
// Listing commands only need source path
|
||||
args = append(args, sourcePath)
|
||||
case "info":
|
||||
// Info commands typically need only source path
|
||||
args = append(args, sourcePath)
|
||||
case "directory":
|
||||
// Directory operations might need one or both paths depending on operation
|
||||
if cmdName == "rmdirs" && strings.Contains(config.RcloneFlags, "--dst") {
|
||||
// Special case: rmdirs with --dst flag needs both paths
|
||||
args = append(args, sourcePath, destPath)
|
||||
} else {
|
||||
// Default case: just source path
|
||||
args = append(args, sourcePath)
|
||||
}
|
||||
case "destructive":
|
||||
// Destructive commands only need source path
|
||||
args = append(args, sourcePath)
|
||||
case "special":
|
||||
// Special commands handled case by case
|
||||
if cmdName == "cryptcheck" {
|
||||
args = append(args, sourcePath, destPath)
|
||||
} else if cmdName == "obscure" || cmdName == "version" || cmdName == "listremotes" {
|
||||
// These commands don't need paths at all
|
||||
} else {
|
||||
args = append(args, sourcePath)
|
||||
}
|
||||
default:
|
||||
// Default to source path only
|
||||
args = append(args, sourcePath)
|
||||
}
|
||||
|
||||
// Execute the command
|
||||
rclonePath := os.Getenv("RCLONE_PATH")
|
||||
if rclonePath == "" {
|
||||
rclonePath = "rclone"
|
||||
}
|
||||
|
||||
te.logger.LogDebug("Full command: %s %v", rclonePath, args)
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
|
||||
// Capture output
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout = &stdout
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
// Start timer for operation
|
||||
startTime := time.Now()
|
||||
|
||||
// Run the command
|
||||
err := cmd.Run()
|
||||
|
||||
// Calculate duration
|
||||
duration := time.Since(startTime)
|
||||
|
||||
// Update history with basic info
|
||||
history.EndTime = &time.Time{}
|
||||
*history.EndTime = startTime.Add(duration)
|
||||
|
||||
// Check for pattern in stderr that indicates successful completion with warnings
|
||||
// Some commands like sync may complete successfully but with warnings
|
||||
successWithWarnings := strings.Contains(stderr.String(), "Transferred:") &&
|
||||
strings.Contains(stderr.String(), "Errors:") &&
|
||||
strings.Contains(stderr.String(), "Checks:")
|
||||
|
||||
// Process results
|
||||
if err != nil && !successWithWarnings {
|
||||
te.logger.LogError("Error executing command '%s' for job %d, config %d: %v", cmdName, job.ID, config.ID, err)
|
||||
te.logger.LogError("Command stderr: %s", stderr.String())
|
||||
|
||||
history.Status = "failed"
|
||||
history.ErrorMessage = fmt.Sprintf("Command Error: %v\nStderr: %s", err, stderr.String())
|
||||
} else {
|
||||
te.logger.LogInfo("Successfully executed command '%s' for job %d, config %d (duration: %v)",
|
||||
cmdName, job.ID, config.ID, duration)
|
||||
|
||||
// Handle different command output types
|
||||
if cmdType == "listing" {
|
||||
// For listing commands, count the number of lines in the output as "files processed"
|
||||
lines := strings.Count(stdout.String(), "\n")
|
||||
history.FilesTransferred = lines
|
||||
history.Status = "completed"
|
||||
} else if cmdType == "transfer" {
|
||||
// Try to extract transfer statistics from command output
|
||||
history.Status = "completed"
|
||||
|
||||
// Look for metrics in stderr which is where rclone puts stats
|
||||
// Extract bytes transferred if available
|
||||
bytesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Bytes`)
|
||||
if matches := bytesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 {
|
||||
if bytesTransferred, err := strconv.ParseInt(matches[1], 10, 64); err == nil {
|
||||
history.BytesTransferred = bytesTransferred
|
||||
}
|
||||
}
|
||||
|
||||
// Extract files transferred if available
|
||||
filesRegex := regexp.MustCompile(`Transferred:\s+(\d+)\s+/\s+(\d+)\s+Files`)
|
||||
if matches := filesRegex.FindStringSubmatch(stderr.String()); len(matches) >= 3 {
|
||||
if filesTransferred, err := strconv.Atoi(matches[1]); err == nil {
|
||||
history.FilesTransferred = filesTransferred
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For other commands, we don't have file counts, but the command completed
|
||||
history.Status = "completed"
|
||||
}
|
||||
|
||||
// Store command output in the history for reference
|
||||
if cmdType == "listing" || cmdType == "info" {
|
||||
// For listing and info commands, the output is the result
|
||||
// Limit to first 1000 characters to avoid huge entries
|
||||
output := stdout.String()
|
||||
if len(output) > 1000 {
|
||||
output = output[:997] + "..."
|
||||
}
|
||||
history.ErrorMessage = fmt.Sprintf("Command Output:\n%s", output)
|
||||
}
|
||||
}
|
||||
|
||||
// Update job history in the database
|
||||
if err := te.db.UpdateJobHistory(history); err != nil {
|
||||
te.logger.LogError("Error updating job history for job %d, config %d: %v", job.ID, config.ID, err)
|
||||
}
|
||||
|
||||
// Send notification
|
||||
te.notifier.SendNotifications(&job, history, &config)
|
||||
}
|
||||
|
||||
// prepareBaseArguments prepares the base arguments for a command
|
||||
func (te *TransferExecutor) prepareBaseArguments(command string, config *db.TransferConfig, progressCallback func(string)) []string {
|
||||
args := []string{command}
|
||||
|
||||
// Add rclone flags from the config
|
||||
if config.CommandFlags != "" {
|
||||
var flagIDs []uint
|
||||
if err := json.Unmarshal([]byte(config.CommandFlags), &flagIDs); err != nil {
|
||||
te.logger.LogError("Error parsing command flags: %v", err)
|
||||
} else {
|
||||
// Get all available flags for this command and their values
|
||||
flagsMap, err := te.db.GetRcloneCommandFlagsMap(config.CommandID)
|
||||
if err != nil {
|
||||
te.logger.LogError("Error getting flags map: %v", err)
|
||||
} else {
|
||||
// Parse flag values if available
|
||||
var flagValues map[uint]string
|
||||
if config.CommandFlagValues != "" {
|
||||
if err := json.Unmarshal([]byte(config.CommandFlagValues), &flagValues); err != nil {
|
||||
te.logger.LogError("Error parsing flag values: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Add each selected flag
|
||||
for _, flagID := range flagIDs {
|
||||
if flag, ok := flagsMap[flagID]; ok {
|
||||
if flag.DataType == "bool" {
|
||||
// Boolean flags don't have values
|
||||
args = append(args, "--"+flag.Name) // Prepend -- for rclone flags
|
||||
} else if flagValues != nil {
|
||||
// Check if we have a value for this flag
|
||||
if value, ok := flagValues[flagID]; ok && value != "" {
|
||||
args = append(args, "--"+flag.Name, value) // Prepend --
|
||||
} else {
|
||||
// If there's a default value, use it
|
||||
if flag.DefaultValue != "" {
|
||||
args = append(args, "--"+flag.Name, flag.DefaultValue) // Prepend --
|
||||
} else {
|
||||
// Skip flags without values
|
||||
te.logger.LogError("Skipping flag %s: no value provided", flag.Name)
|
||||
}
|
||||
}
|
||||
} else if flag.DefaultValue != "" { // Handle case where flagValues is nil but default exists
|
||||
args = append(args, "--"+flag.Name, flag.DefaultValue) // Prepend --
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add any additional rclone flags specified by the user
|
||||
if config.RcloneFlags != "" {
|
||||
additionalFlags := strings.Fields(config.RcloneFlags)
|
||||
args = append(args, additionalFlags...)
|
||||
}
|
||||
|
||||
// Add common rclone options
|
||||
args = append(args, "--progress")
|
||||
args = append(args, "--stats", "1s")
|
||||
|
||||
// Add config file location
|
||||
configPath := te.db.GetConfigRclonePath(config)
|
||||
args = append(args, "--config", configPath)
|
||||
|
||||
// Add progress callback related flags if needed (progressCallback is currently nil)
|
||||
if progressCallback != nil {
|
||||
args = append(args, "--stats-one-line")
|
||||
// Potentially add --json if parsing progress
|
||||
} else {
|
||||
// Default behavior without callback
|
||||
args = append(args, "--stats-one-line") // Keep this for general stats output
|
||||
}
|
||||
|
||||
// Consider adding --json only if specifically needed for parsing output later
|
||||
// args = append(args, "--json")
|
||||
|
||||
return args
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ProcessOutputPattern processes an output pattern with variables and returns the result
|
||||
// This function is useful for testing pattern processing in isolation
|
||||
func ProcessOutputPattern(pattern string, originalFilename string) string {
|
||||
// Process date variables
|
||||
dateRegex := regexp.MustCompile(`\${date:([^}]+)}`)
|
||||
processedPattern := dateRegex.ReplaceAllStringFunc(pattern, func(match string) string {
|
||||
format := dateRegex.FindStringSubmatch(match)[1]
|
||||
return time.Now().Format(format)
|
||||
})
|
||||
|
||||
// Split the filename and extension
|
||||
ext := filepath.Ext(originalFilename)
|
||||
filename := strings.TrimSuffix(originalFilename, ext)
|
||||
|
||||
// Replace filename and extension variables
|
||||
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", filename)
|
||||
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", ext)
|
||||
|
||||
return processedPattern
|
||||
}
|
||||
|
||||
// createRcloneFilterFile creates a temporary filter file for rclone with rename rules
|
||||
func createRcloneFilterFile(pattern string) (string, error) {
|
||||
// Create a temporary file
|
||||
tmpFile, err := ioutil.TempFile("", "rclone-filter-*.txt")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create temporary filter file: %v", err)
|
||||
}
|
||||
defer tmpFile.Close()
|
||||
|
||||
// Process the pattern to create a rclone filter rule
|
||||
// First, replace date variables with current date in the specified format
|
||||
dateRegex := regexp.MustCompile(`\${date:([^}]+)}`)
|
||||
processedPattern := dateRegex.ReplaceAllStringFunc(pattern, func(match string) string {
|
||||
format := dateRegex.FindStringSubmatch(match)[1]
|
||||
return time.Now().Format(format)
|
||||
})
|
||||
|
||||
// Replace filename and extension variables with rclone's capture group references
|
||||
// For rclone rename filters, we need to use {1} for the first capture group, not $1
|
||||
// See: https://rclone.org/filtering/#rename
|
||||
|
||||
// Extract filename without extension
|
||||
processedPattern = strings.ReplaceAll(processedPattern, "${filename}", "{1}")
|
||||
|
||||
// Extract extension (with the dot)
|
||||
processedPattern = strings.ReplaceAll(processedPattern, "${ext}", "{2}")
|
||||
|
||||
// Create a rename rule for rclone using the correct syntax:
|
||||
// - The format for rename filters is: "-- SourceRegexp ReplacementPattern"
|
||||
// - For files with extension: capture the name and extension separately
|
||||
rule := fmt.Sprintf("-- (.*)(\\..+)$ %s\n", processedPattern)
|
||||
|
||||
// Add a fallback rule for files without extension
|
||||
fallbackRule := fmt.Sprintf("-- ([^.]+)$ %s\n",
|
||||
strings.ReplaceAll(processedPattern, "{2}", ""))
|
||||
|
||||
// Write the rules to the file
|
||||
if _, err := tmpFile.WriteString(rule + fallbackRule); err != nil {
|
||||
return "", fmt.Errorf("failed to write to filter file: %v", err)
|
||||
}
|
||||
|
||||
return tmpFile.Name(), nil
|
||||
}
|
||||
@@ -1,92 +0,0 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/glebarez/sqlite"
|
||||
"github.com/starfleetcptn/gomft/internal/db"
|
||||
"github.com/starfleetcptn/gomft/internal/email"
|
||||
"github.com/starfleetcptn/gomft/internal/scheduler"
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// Static counter to ensure unique emails for each test
|
||||
var testEmailCounter int = 0
|
||||
|
||||
func setupTestHandlers(t *testing.T) (*Handlers, *gin.Engine) {
|
||||
// Set Gin to test mode
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
// Create a test DB
|
||||
testDB := setupTestDB(t)
|
||||
|
||||
// Create a mock scheduler
|
||||
mockScheduler := &scheduler.Scheduler{}
|
||||
|
||||
// Create a mock email service
|
||||
mockEmailService := &email.Service{}
|
||||
|
||||
// Create test handlers
|
||||
handlers := NewHandlers(
|
||||
testDB,
|
||||
mockScheduler,
|
||||
"test-jwt-secret",
|
||||
"test-db-path",
|
||||
"test-backup-dir",
|
||||
"test-logs-dir",
|
||||
mockEmailService,
|
||||
)
|
||||
|
||||
// Create a test router
|
||||
router := gin.New()
|
||||
|
||||
return handlers, router
|
||||
}
|
||||
|
||||
// setupTestDB creates a test database for handler tests
|
||||
func setupTestDB(t *testing.T) *db.DB {
|
||||
// Set up an in-memory SQLite DB
|
||||
gormDB, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to open in-memory database: %v", err)
|
||||
}
|
||||
|
||||
// Run migrations
|
||||
err = gormDB.AutoMigrate(
|
||||
&db.User{},
|
||||
&db.PasswordHistory{},
|
||||
&db.PasswordResetToken{},
|
||||
&db.TransferConfig{},
|
||||
&db.Job{},
|
||||
&db.JobHistory{},
|
||||
&db.FileMetadata{},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to migrate database: %v", err)
|
||||
}
|
||||
|
||||
// Create a test admin user with a unique email
|
||||
testEmailCounter++
|
||||
testEmail := fmt.Sprintf("test%d@example.com", testEmailCounter)
|
||||
|
||||
// Generate a hashed password for "admin"
|
||||
hashedPassword, err := bcrypt.GenerateFromPassword([]byte("admin"), bcrypt.DefaultCost)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to hash password: %v", err)
|
||||
}
|
||||
|
||||
admin := db.User{
|
||||
Email: testEmail,
|
||||
PasswordHash: string(hashedPassword),
|
||||
}
|
||||
admin.SetIsAdmin(true)
|
||||
|
||||
if err := gormDB.Create(&admin).Error; err != nil {
|
||||
t.Fatalf("Failed to create test admin user: %v", err)
|
||||
}
|
||||
|
||||
return &db.DB{DB: gormDB}
|
||||
}
|
||||
@@ -15,102 +15,6 @@ import (
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
// HandleUsers handles the GET /admin/users route
|
||||
// func (h *Handlers) HandleUsers(c *gin.Context) {
|
||||
// var users []db.User
|
||||
// if err := h.DB.Find(&users).Error; err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to retrieve users")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create the data for the Users component, not UserManagement
|
||||
// data := components.UsersData{
|
||||
// Users: users,
|
||||
// }
|
||||
|
||||
// // Use the Users component from users.templ and ensure context flows through consistently
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// components.Users(ctx, data).Render(ctx, c.Writer)
|
||||
// }
|
||||
|
||||
// HandleNewUser handles the GET /admin/users/new route
|
||||
// func (h *Handlers) HandleNewUser(c *gin.Context) {
|
||||
// // Create the data for the UserForm component
|
||||
// data := components.UserFormData{
|
||||
// IsNew: true,
|
||||
// ErrorMessage: "",
|
||||
// }
|
||||
|
||||
// // Use consistent context handling
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// err := components.UserForm(ctx, data).Render(ctx, c.Writer)
|
||||
// if err != nil {
|
||||
// log.Printf("ERROR rendering UserForm: %v", err)
|
||||
// c.String(http.StatusInternalServerError, "Error rendering form: %v", err)
|
||||
// return
|
||||
// }
|
||||
// }
|
||||
|
||||
// HandleCreateUser handles the POST /admin/users/new route
|
||||
// func (h *Handlers) HandleCreateUser(c *gin.Context) {
|
||||
// email := c.PostForm("email")
|
||||
// password := c.PostForm("password")
|
||||
// isAdmin := c.PostForm("is_admin") == "on"
|
||||
|
||||
// // Check if email already exists
|
||||
// var existingUser db.User
|
||||
// if err := h.DB.Where("email = ?", email).First(&existingUser).Error; err == nil {
|
||||
// c.String(http.StatusBadRequest, "Email already exists")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Hash the password
|
||||
// hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost)
|
||||
// if err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to hash password")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create the user
|
||||
// user := db.User{
|
||||
// Email: email,
|
||||
// PasswordHash: string(hashedPassword),
|
||||
// LastPasswordChange: time.Now(),
|
||||
// }
|
||||
// user.SetIsAdmin(isAdmin)
|
||||
|
||||
// if err := h.DB.Create(&user).Error; err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to create user")
|
||||
// return
|
||||
// }
|
||||
|
||||
// c.Redirect(http.StatusSeeOther, "/admin/users")
|
||||
// }
|
||||
|
||||
// HandleDeleteUser handles the POST /admin/users/delete route
|
||||
// func (h *Handlers) HandleDeleteUser(c *gin.Context) {
|
||||
// userID, err := strconv.ParseUint(c.Param("id"), 10, 32)
|
||||
// if err != nil {
|
||||
// c.String(http.StatusBadRequest, "Invalid user ID")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Don't allow deleting the current user
|
||||
// currentUserID := c.GetUint("userID")
|
||||
// if uint(userID) == currentUserID {
|
||||
// c.String(http.StatusBadRequest, "Cannot delete your own account")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Delete the user
|
||||
// if err := h.DB.Delete(&db.User{}, userID).Error; err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to delete user")
|
||||
// return
|
||||
// }
|
||||
|
||||
// c.Redirect(http.StatusSeeOther, "/admin/users")
|
||||
// }
|
||||
|
||||
// HandleRegisterPage handles the GET /register route
|
||||
func (h *Handlers) HandleRegisterPage(c *gin.Context) {
|
||||
// Check if any users exist
|
||||
@@ -303,39 +207,6 @@ func (h *Handlers) AdminCreateUser(c *gin.Context) {
|
||||
c.Redirect(http.StatusFound, "/admin/users")
|
||||
}
|
||||
|
||||
// HandleEditUser handles the GET /admin/users/:id/edit route
|
||||
// func (h *Handlers) HandleEditUser(c *gin.Context) {
|
||||
// id := c.Param("id")
|
||||
|
||||
// var user db.User
|
||||
// if err := h.DB.First(&user, id).Error; err != nil {
|
||||
// c.Redirect(http.StatusFound, "/admin/users")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Get user's roles
|
||||
// userRoles, err := h.DB.GetUserRoles(user.ID)
|
||||
// if err != nil {
|
||||
// log.Printf("Error fetching user roles: %v", err)
|
||||
// }
|
||||
|
||||
// // Get all available roles
|
||||
// var allRoles []db.Role
|
||||
// if err := h.DB.Find(&allRoles).Error; err != nil {
|
||||
// log.Printf("Error fetching roles: %v", err)
|
||||
// }
|
||||
|
||||
// // Use UserEdit component to match the form that's being submitted
|
||||
// data := components.UserEditData{
|
||||
// User: &user,
|
||||
// Roles: allRoles,
|
||||
// UserRoles: userRoles,
|
||||
// IsNew: false,
|
||||
// }
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// components.UserEdit(ctx, data).Render(ctx, c.Writer)
|
||||
// }
|
||||
|
||||
// AdminUpdateUser handles the PUT /admin/users/:id route
|
||||
func (h *Handlers) AdminUpdateUser(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
@@ -569,307 +440,6 @@ func (h *Handlers) AdminDeleteUser(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"message": "User deleted successfully"})
|
||||
}
|
||||
|
||||
// AdminRoles handles the GET /admin/roles route
|
||||
// func (h *Handlers) AdminRoles(c *gin.Context) {
|
||||
// var dbRoles []db.Role
|
||||
// if err := h.DB.Find(&dbRoles).Error; err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Error fetching roles")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Convert db.Role to components.Role
|
||||
// var roles []components.Role
|
||||
// for _, dbRole := range dbRoles {
|
||||
// role := components.Role{
|
||||
// ID: dbRole.ID,
|
||||
// Name: dbRole.Name,
|
||||
// Description: dbRole.Description,
|
||||
// Permissions: dbRole.Permissions,
|
||||
// }
|
||||
// roles = append(roles, role)
|
||||
// }
|
||||
|
||||
// // Use components instead of HTML templates
|
||||
// data := components.RolesData{
|
||||
// Roles: roles,
|
||||
// }
|
||||
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// components.AdminRoles(ctx, data).Render(ctx, c.Writer)
|
||||
// }
|
||||
|
||||
// AdminNewRolePage handles the GET /admin/roles/new route
|
||||
// func (h *Handlers) AdminNewRolePage(c *gin.Context) {
|
||||
// // Create an empty role for the form
|
||||
// role := &components.Role{
|
||||
// ID: 0,
|
||||
// Name: "",
|
||||
// Description: "",
|
||||
// Permissions: []string{},
|
||||
// }
|
||||
|
||||
// // All available permissions
|
||||
// allPermissions := []string{
|
||||
// "users.view", "users.create", "users.edit", "users.delete",
|
||||
// "roles.view", "roles.create", "roles.edit", "roles.delete",
|
||||
// "transfers.view", "transfers.create", "transfers.edit", "transfers.delete",
|
||||
// "audit.view",
|
||||
// }
|
||||
|
||||
// // Use components instead of HTML templates
|
||||
// data := components.RoleFormData{
|
||||
// Role: role,
|
||||
// IsNew: true,
|
||||
// AllPermissions: allPermissions,
|
||||
// }
|
||||
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// components.AdminRoleForm(ctx, data).Render(ctx, c.Writer)
|
||||
// }
|
||||
|
||||
// AdminCreateRole handles the POST /admin/roles route
|
||||
// func (h *Handlers) AdminCreateRole(c *gin.Context) {
|
||||
// name := c.PostForm("name")
|
||||
// description := c.PostForm("description")
|
||||
// permissions := c.PostFormArray("permissions[]")
|
||||
|
||||
// // Create role
|
||||
// role := &db.Role{
|
||||
// Name: name,
|
||||
// Description: description,
|
||||
// }
|
||||
// role.SetPermissions(permissions)
|
||||
|
||||
// // Validate role
|
||||
// if err := role.Validate(); err != nil {
|
||||
// ctx := components.CreateTemplateContext(c)
|
||||
// data := components.RoleFormData{
|
||||
// Role: &components.Role{Name: name, Description: description, Permissions: permissions},
|
||||
// IsNew: true,
|
||||
// ErrorMessage: err.Error(),
|
||||
// AllPermissions: GetAllPermissions(),
|
||||
// }
|
||||
// _ = components.AdminRoleForm(ctx, data).Render(ctx, c.Writer)
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Start transaction
|
||||
// tx := h.DB.Begin()
|
||||
// if err := tx.Error; err != nil {
|
||||
// handleRoleError(c, role, true, "Failed to begin transaction: "+err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create role in database
|
||||
// if err := tx.Create(role).Error; err != nil {
|
||||
// tx.Rollback()
|
||||
// handleRoleError(c, role, true, "Failed to create role: "+err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create audit log
|
||||
// userID := getUserID(c) // Implement this helper to get current user ID
|
||||
// if err := role.AuditLog(tx, "create", userID); err != nil {
|
||||
// tx.Rollback()
|
||||
// handleRoleError(c, role, true, "Failed to create audit log: "+err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Commit transaction
|
||||
// if err := tx.Commit().Error; err != nil {
|
||||
// handleRoleError(c, role, true, "Failed to commit transaction: "+err.Error())
|
||||
// return
|
||||
// }
|
||||
|
||||
// c.Redirect(http.StatusFound, "/admin/roles")
|
||||
// }
|
||||
|
||||
// AdminEditRolePage handles the GET /admin/roles/:id/edit route
|
||||
// func (h *Handlers) AdminEditRolePage(c *gin.Context) {
|
||||
// id := c.Param("id")
|
||||
|
||||
// var dbRole db.Role
|
||||
// if err := h.DB.First(&dbRole, id).Error; err != nil {
|
||||
// c.Redirect(http.StatusFound, "/admin/roles")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Convert db.Role to components.Role
|
||||
// role := &components.Role{
|
||||
// ID: dbRole.ID,
|
||||
// Name: dbRole.Name,
|
||||
// Description: dbRole.Description,
|
||||
// Permissions: dbRole.Permissions,
|
||||
// }
|
||||
|
||||
// // All available permissions
|
||||
// allPermissions := []string{
|
||||
// "users.view", "users.create", "users.edit", "users.delete",
|
||||
// "roles.view", "roles.create", "roles.edit", "roles.delete",
|
||||
// "transfers.view", "transfers.create", "transfers.edit", "transfers.delete",
|
||||
// "audit.view",
|
||||
// }
|
||||
|
||||
// // Use components instead of HTML templates
|
||||
// data := components.RoleFormData{
|
||||
// Role: role,
|
||||
// IsNew: false,
|
||||
// AllPermissions: allPermissions,
|
||||
// }
|
||||
|
||||
// ctx := h.CreateTemplateContext(c)
|
||||
// components.AdminRoleForm(ctx, data).Render(ctx, c.Writer)
|
||||
// }
|
||||
|
||||
// AdminUpdateRole handles the PUT /admin/roles/:id route
|
||||
// func (h *Handlers) AdminUpdateRole(c *gin.Context) {
|
||||
// id := c.Param("id")
|
||||
|
||||
// var role db.Role
|
||||
// if err := h.DB.First(&role, id).Error; err != nil {
|
||||
// c.String(http.StatusNotFound, "Role not found")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Store original role state for audit log
|
||||
// oldRole := role
|
||||
|
||||
// // Update role with form data
|
||||
// name := c.PostForm("name")
|
||||
// if name != "" {
|
||||
// role.Name = name
|
||||
// }
|
||||
|
||||
// description := c.PostForm("description")
|
||||
// if description != "" {
|
||||
// role.Description = description
|
||||
// }
|
||||
|
||||
// // Process permissions
|
||||
// permissionsStr := c.PostForm("permissions")
|
||||
// if permissionsStr != "" {
|
||||
// permissions := strings.Split(permissionsStr, ",")
|
||||
// for i, p := range permissions {
|
||||
// permissions[i] = strings.TrimSpace(p)
|
||||
// }
|
||||
// role.Permissions = db.Permissions(permissions)
|
||||
// }
|
||||
|
||||
// // Start a transaction
|
||||
// tx := h.DB.Begin()
|
||||
// if tx.Error != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to begin transaction")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Save the role
|
||||
// if err := tx.Save(&role).Error; err != nil {
|
||||
// tx.Rollback()
|
||||
// c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to update role: %v", err))
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create audit log entry
|
||||
// adminID := c.GetUint("userID")
|
||||
// auditDetails := map[string]interface{}{
|
||||
// "name": role.Name,
|
||||
// "description": role.Description,
|
||||
// "permissions": role.Permissions,
|
||||
// "previous_state": map[string]interface{}{
|
||||
// "name": oldRole.Name,
|
||||
// "description": oldRole.Description,
|
||||
// "permissions": oldRole.Permissions,
|
||||
// },
|
||||
// }
|
||||
|
||||
// auditLog := db.AuditLog{
|
||||
// Action: "update",
|
||||
// EntityType: "role",
|
||||
// EntityID: role.ID,
|
||||
// UserID: adminID,
|
||||
// Details: auditDetails,
|
||||
// Timestamp: time.Now(),
|
||||
// }
|
||||
|
||||
// if err := tx.Create(&auditLog).Error; err != nil {
|
||||
// tx.Rollback()
|
||||
// c.String(http.StatusInternalServerError, "Failed to create audit log")
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Commit the transaction
|
||||
// if err := tx.Commit().Error; err != nil {
|
||||
// c.String(http.StatusInternalServerError, "Failed to commit transaction")
|
||||
// return
|
||||
// }
|
||||
|
||||
// c.Redirect(http.StatusFound, "/admin/roles")
|
||||
// }
|
||||
|
||||
// AdminDeleteRole handles the DELETE /admin/roles/:id route
|
||||
// func (h *Handlers) AdminDeleteRole(c *gin.Context) {
|
||||
// id := c.Param("id")
|
||||
// adminID := c.GetUint("userID")
|
||||
|
||||
// var role db.Role
|
||||
// if err := h.DB.First(&role, id).Error; err != nil {
|
||||
// c.JSON(http.StatusNotFound, gin.H{"error": "Role not found"})
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Check if role is in use
|
||||
// var count int64
|
||||
// h.DB.Table("user_roles").Where("role_id = ?", role.ID).Count(&count)
|
||||
// if count > 0 {
|
||||
// c.JSON(http.StatusBadRequest, gin.H{"error": "Role is assigned to users and cannot be deleted"})
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Start a transaction
|
||||
// tx := h.DB.Begin()
|
||||
// if tx.Error != nil {
|
||||
// c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to begin transaction"})
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Create audit log before deletion
|
||||
// auditDetails := map[string]interface{}{
|
||||
// "name": role.Name,
|
||||
// "description": role.Description,
|
||||
// "permissions": role.Permissions,
|
||||
// }
|
||||
|
||||
// auditLog := db.AuditLog{
|
||||
// Action: "delete",
|
||||
// EntityType: "role",
|
||||
// EntityID: role.ID,
|
||||
// UserID: adminID,
|
||||
// Details: auditDetails,
|
||||
// Timestamp: time.Now(),
|
||||
// }
|
||||
|
||||
// if err := tx.Create(&auditLog).Error; err != nil {
|
||||
// tx.Rollback()
|
||||
// c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to create audit log"})
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Delete the role
|
||||
// if err := tx.Delete(&role).Error; err != nil {
|
||||
// tx.Rollback()
|
||||
// c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Failed to delete role: %v", err)})
|
||||
// return
|
||||
// }
|
||||
|
||||
// // Commit the transaction
|
||||
// if err := tx.Commit().Error; err != nil {
|
||||
// c.JSON(http.StatusInternalServerError, gin.H{"error": "Failed to commit transaction"})
|
||||
// return
|
||||
// }
|
||||
|
||||
// c.JSON(http.StatusOK, gin.H{"message": "Role deleted successfully"})
|
||||
// }
|
||||
|
||||
// AdminUserRoles handles the GET /admin/users/:id/roles route
|
||||
func (h *Handlers) AdminUserRoles(c *gin.Context) {
|
||||
id := c.Param("id")
|
||||
|
||||
Reference in New Issue
Block a user