mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-11-17 17:24:43 +01:00
feat: Enhance notification service configuration and refactor related components
- Updated notification service models to use pointers for boolean fields, allowing for better handling of enabled states. - Introduced helper methods for getting and setting enabled states in notification and auth provider models. - Refactored notification form templates to consolidate event trigger handling, improving code maintainability. - Added SSL verification configuration to the application settings, allowing users to skip SSL verification for outgoing notifications. - Enhanced the notification sending logic to respect the SSL verification setting. - Introduced new migration to alter boolean defaults in the database schema for better compatibility with the updated models. - Added unit tests for email and Rclone service functionalities to ensure reliability.
This commit is contained in:
1 parent
b35174f857
commit
608da9ce1c
34 files changed
+1230
-896
No files matched your search
@@ -28,7 +28,7 @@ type AuthProvider struct {
|
||||
ID uint `gorm:"primarykey" json:"id"`
|
||||
Name string `gorm:"not null" json:"name"`
|
||||
Type ProviderType `gorm:"not null" json:"type"`
|
||||
Enabled bool `gorm:"default:true" json:"enabled"`
|
||||
Enabled *bool `gorm:"default:true" json:"enabled"`
|
||||
Description string `json:"description"`
|
||||
ProviderURL string `json:"provider_url"`
|
||||
ClientID string `json:"client_id"`
|
||||
@@ -75,6 +75,21 @@ func (p *AuthProvider) SetConfig(data map[string]interface{}) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- AuthProvider Helper Methods ---
|
||||
|
||||
// GetEnabled returns the value of Enabled with a default if nil
|
||||
func (p *AuthProvider) GetEnabled() bool {
|
||||
if p.Enabled == nil {
|
||||
return true // Default to true if not set
|
||||
}
|
||||
return *p.Enabled
|
||||
}
|
||||
|
||||
// SetEnabled sets the Enabled field
|
||||
func (p *AuthProvider) SetEnabled(value bool) {
|
||||
p.Enabled = &value
|
||||
}
|
||||
|
||||
// ExternalUserIdentity represents a user identity from an external authentication provider
|
||||
type ExternalUserIdentity struct {
|
||||
ID uint `gorm:"primarykey" json:"id"`
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AlterBooleanDefaults changes boolean columns with default:true to pointers
|
||||
// using explicit table recreation with raw SQL for SQLite compatibility.
|
||||
func AlterBooleanDefaults() *gormigrate.Migration {
|
||||
|
||||
// --- Raw SQL CREATE TABLE statements for the target schema ---
|
||||
|
||||
const createNotificationServicesSQL = `
|
||||
CREATE TABLE notification_services (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
is_enabled INTEGER DEFAULT 1, -- Target: *bool, SQLite uses 0/1, default true
|
||||
config TEXT,
|
||||
description TEXT,
|
||||
event_triggers TEXT DEFAULT '[]',
|
||||
payload_template TEXT,
|
||||
secret_key TEXT,
|
||||
retry_policy TEXT DEFAULT 'simple',
|
||||
last_used timestamp,
|
||||
success_count INTEGER DEFAULT 0,
|
||||
failure_count INTEGER DEFAULT 0,
|
||||
created_by INTEGER,
|
||||
created_at timestamp,
|
||||
updated_at timestamp
|
||||
);`
|
||||
|
||||
const createAuthProvidersSQL = `
|
||||
CREATE TABLE auth_providers (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
type TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 1, -- Target: *bool, SQLite uses 0/1, default true
|
||||
description TEXT,
|
||||
provider_url TEXT,
|
||||
client_id TEXT,
|
||||
client_secret TEXT,
|
||||
redirect_url TEXT,
|
||||
scopes TEXT,
|
||||
attribute_mapping TEXT,
|
||||
config TEXT,
|
||||
icon_url TEXT,
|
||||
successful_logins INTEGER DEFAULT 0,
|
||||
last_used timestamp,
|
||||
created_at timestamp,
|
||||
updated_at timestamp
|
||||
);`
|
||||
|
||||
const createTransferConfigsSQL = `
|
||||
CREATE TABLE transfer_configs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
source_type TEXT NOT NULL,
|
||||
source_path TEXT NOT NULL,
|
||||
source_host TEXT,
|
||||
source_port INTEGER DEFAULT 22,
|
||||
source_user TEXT,
|
||||
source_key_file TEXT,
|
||||
source_bucket TEXT,
|
||||
source_region TEXT,
|
||||
source_access_key TEXT,
|
||||
source_endpoint TEXT,
|
||||
source_share TEXT,
|
||||
source_domain TEXT,
|
||||
source_passive_mode INTEGER DEFAULT 1, -- Already *bool, keep default
|
||||
source_client_id TEXT,
|
||||
source_drive_id TEXT,
|
||||
source_team_drive TEXT,
|
||||
source_read_only INTEGER,
|
||||
source_start_year INTEGER,
|
||||
source_include_archived INTEGER,
|
||||
file_pattern TEXT DEFAULT '*',
|
||||
output_pattern TEXT,
|
||||
destination_type TEXT NOT NULL,
|
||||
destination_path TEXT NOT NULL,
|
||||
dest_host TEXT,
|
||||
dest_port INTEGER DEFAULT 22,
|
||||
dest_user TEXT,
|
||||
dest_key_file TEXT,
|
||||
dest_bucket TEXT,
|
||||
dest_region TEXT,
|
||||
dest_access_key TEXT,
|
||||
dest_endpoint TEXT,
|
||||
dest_share TEXT,
|
||||
dest_domain TEXT,
|
||||
dest_passive_mode INTEGER DEFAULT 1, -- Already *bool, keep default
|
||||
dest_client_id TEXT,
|
||||
dest_drive_id TEXT,
|
||||
dest_team_drive TEXT,
|
||||
dest_read_only INTEGER,
|
||||
dest_start_year INTEGER,
|
||||
dest_include_archived INTEGER,
|
||||
use_builtin_auth_source INTEGER,
|
||||
use_builtin_auth_dest INTEGER,
|
||||
google_drive_authenticated INTEGER,
|
||||
archive_path TEXT,
|
||||
archive_enabled INTEGER DEFAULT 0,
|
||||
rclone_flags TEXT,
|
||||
command_id INTEGER DEFAULT 1,
|
||||
command_flags TEXT,
|
||||
command_flag_values TEXT,
|
||||
delete_after_transfer INTEGER DEFAULT 0,
|
||||
skip_processed_files INTEGER DEFAULT 1, -- Target: *bool, SQLite uses 0/1, default true
|
||||
max_concurrent_transfers INTEGER DEFAULT 4,
|
||||
created_by INTEGER,
|
||||
created_at timestamp,
|
||||
updated_at timestamp
|
||||
);`
|
||||
|
||||
// --- End Raw SQL ---
|
||||
|
||||
return &gormigrate.Migration{
|
||||
ID: "012_alter_boolean_defaults",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// --- Backup Logic (copied) ---
|
||||
var count int64
|
||||
if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil {
|
||||
return fmt.Errorf("failed to check for existing tables: %v", err)
|
||||
}
|
||||
if count > 0 {
|
||||
sqlDB, err := tx.DB()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get underlying database: %v", err)
|
||||
}
|
||||
var seq int
|
||||
var name, dbPath string
|
||||
if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil {
|
||||
return fmt.Errorf("failed to get database path: %v", err)
|
||||
}
|
||||
backupDir := os.Getenv("BACKUP_DIR")
|
||||
if backupDir == "" {
|
||||
backupDir = "/app/backups"
|
||||
if _, err := os.Stat(backupDir); os.IsNotExist(err) {
|
||||
backupDir = "backups"
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(backupDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create backup directory: %v", err)
|
||||
}
|
||||
dbFileName := filepath.Base(dbPath)
|
||||
backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405"))
|
||||
backupFile := filepath.Join(backupDir, backupFileName)
|
||||
data, err := os.ReadFile(dbPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read database for backup: %v", err)
|
||||
}
|
||||
if err := os.WriteFile(backupFile, data, 0600); err != nil {
|
||||
return fmt.Errorf("failed to create database backup: %v", err)
|
||||
}
|
||||
fmt.Printf("Created database backup at: %s\n", backupFile)
|
||||
}
|
||||
// --- End Backup Logic ---
|
||||
|
||||
// --- Table Recreation Logic for SQLite ---
|
||||
if err := tx.Exec("PRAGMA foreign_keys = OFF").Error; err != nil {
|
||||
return fmt.Errorf("failed to disable foreign keys: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
if err := tx.Exec("PRAGMA foreign_keys = ON").Error; err != nil {
|
||||
fmt.Printf("Warning: failed to re-enable foreign keys: %v\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
// Helper function for table recreation
|
||||
recreateTable := func(tableName, createSQL string) error {
|
||||
fmt.Printf("Recreating table %s...\n", tableName)
|
||||
oldTableName := fmt.Sprintf("_%s_old", tableName)
|
||||
|
||||
// Rename old table
|
||||
if err := tx.Exec(fmt.Sprintf("ALTER TABLE %s RENAME TO %s", tableName, oldTableName)).Error; err == nil {
|
||||
fmt.Printf("Renamed %s to %s.\n", tableName, oldTableName)
|
||||
|
||||
// Create new table using raw SQL
|
||||
fmt.Printf("Creating new %s table...\n", tableName)
|
||||
if err := tx.Exec(createSQL).Error; err != nil {
|
||||
return fmt.Errorf("failed to create new %s table: %w", tableName, err)
|
||||
}
|
||||
fmt.Printf("New %s table created.\n", tableName)
|
||||
|
||||
// Copy data
|
||||
fmt.Printf("Copying data to new %s table...\n", tableName)
|
||||
// IMPORTANT: Ensure column order/names match if schema changed beyond types/defaults
|
||||
if err := tx.Exec(fmt.Sprintf("INSERT INTO %s SELECT * FROM %s", tableName, oldTableName)).Error; err != nil {
|
||||
return fmt.Errorf("failed to copy data to new %s table: %w", tableName, err)
|
||||
}
|
||||
fmt.Printf("Data copied to %s.\n", tableName)
|
||||
|
||||
// Drop old table
|
||||
if err := tx.Exec(fmt.Sprintf("DROP TABLE %s", oldTableName)).Error; err != nil {
|
||||
return fmt.Errorf("failed to drop old %s table: %w", tableName, err)
|
||||
}
|
||||
fmt.Printf("Successfully recreated %s.\n", tableName)
|
||||
} else {
|
||||
// Check if rename failed because table doesn't exist (fresh install)
|
||||
var tableExists int
|
||||
tx.Raw(fmt.Sprintf("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='%s'", tableName)).Scan(&tableExists)
|
||||
if tableExists == 0 {
|
||||
fmt.Printf("%s table does not exist, creating.\n", tableName)
|
||||
if err := tx.Exec(createSQL).Error; err != nil { // Create table directly
|
||||
return fmt.Errorf("failed to create new %s table: %w", tableName, err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("failed to rename %s: %w", tableName, err) // Real rename error
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Recreate tables
|
||||
if err := recreateTable("notification_services", createNotificationServicesSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recreateTable("auth_providers", createAuthProvidersSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := recreateTable("transfer_configs", createTransferConfigsSQL); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// --- End Table Recreation Logic ---
|
||||
|
||||
// Create audit log entry
|
||||
now := time.Now()
|
||||
details, auditErr := json.Marshal(map[string]interface{}{
|
||||
"tables_affected": []string{"notification_services", "auth_providers", "transfer_configs"},
|
||||
"columns_altered": []string{"is_enabled", "enabled", "skip_processed_files"},
|
||||
"new_type": "*bool (pointer to boolean)",
|
||||
"method": "Table recreation (SQLite - Raw SQL)",
|
||||
"message": "Changed boolean columns with default:true to pointers to handle false values correctly with GORM.",
|
||||
})
|
||||
if auditErr != nil {
|
||||
fmt.Printf("Warning: Failed to marshal audit log details: %v\n", auditErr)
|
||||
}
|
||||
|
||||
if auditErr == nil {
|
||||
if auditExecErr := tx.Exec(`
|
||||
INSERT INTO audit_logs (action, entity_type, entity_id, user_id, details, created_at, updated_at, timestamp)
|
||||
VALUES ('schema_update', 'multiple_tables', 0, 1, ?, ?, ?, ?)
|
||||
`, string(details), now, now, now).Error; auditExecErr != nil {
|
||||
fmt.Printf("Warning: Failed to insert audit log: %v\n", auditExecErr)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// Rollback is complex and risky with table recreation. Log skip.
|
||||
now := time.Now()
|
||||
details, err := json.Marshal(map[string]interface{}{
|
||||
"migration_id": "012_alter_boolean_defaults",
|
||||
"message": "Skipping rollback of boolean column type changes (via table recreation) due to complexity/potential data loss.",
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Printf("Warning: Failed to marshal rollback audit log details: %v\n", err)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
if auditExecErr := tx.Exec(`
|
||||
INSERT INTO audit_logs (action, entity_type, entity_id, user_id, details, created_at, updated_at, timestamp)
|
||||
VALUES ('migration_rollback', 'multiple_tables', 0, 1, ?, ?, ?, ?)
|
||||
`, string(details), now, now, now).Error; auditExecErr != nil {
|
||||
fmt.Printf("Warning: Failed to insert rollback audit log: %v\n", auditExecErr)
|
||||
}
|
||||
}
|
||||
fmt.Println("Rollback for migration 012_alter_boolean_defaults skipped for safety.")
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate {
|
||||
AddRcloneTables(), // 009
|
||||
AddRcloneCommandToConfig(), // 010
|
||||
AddAuthProviders(), // 011
|
||||
AlterBooleanDefaults(), // 012
|
||||
)
|
||||
|
||||
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
|
||||
|
||||
@@ -12,7 +12,7 @@ type NotificationService struct {
|
||||
ID uint `json:"id" gorm:"primaryKey"`
|
||||
Name string `json:"name" gorm:"not null"`
|
||||
Type string `json:"type" gorm:"not null"` // email, webhook
|
||||
IsEnabled bool `json:"is_enabled" gorm:"default:true"`
|
||||
IsEnabled *bool `json:"is_enabled" gorm:"default:true"`
|
||||
Config map[string]string `json:"config" gorm:"-"`
|
||||
ConfigJSON string `json:"-" gorm:"column:config"`
|
||||
Description string `json:"description"`
|
||||
@@ -69,6 +69,8 @@ func (db *DB) GetNotificationServices(onlyEnabled bool) ([]NotificationService,
|
||||
query := db.DB
|
||||
|
||||
if onlyEnabled {
|
||||
// When using a pointer, we need to explicitly check for true
|
||||
// GORM handles the underlying SQL correctly for different dialects
|
||||
query = query.Where("is_enabled = ?", true)
|
||||
}
|
||||
|
||||
@@ -102,3 +104,21 @@ func (db *DB) UpdateNotificationService(service *NotificationService) error {
|
||||
func (db *DB) DeleteNotificationService(id uint) error {
|
||||
return db.Delete(&NotificationService{}, id).Error
|
||||
}
|
||||
|
||||
// --- NotificationService Helper Methods ---
|
||||
|
||||
// GetIsEnabled returns the value of IsEnabled with a default if nil
|
||||
func (n *NotificationService) GetIsEnabled() bool {
|
||||
if n.IsEnabled == nil {
|
||||
// If the pointer is nil, GORM might not have set it,
|
||||
// or it was explicitly set to nil. We assume the DB default (true)
|
||||
// if it's nil, aligning with the original gorm tag default.
|
||||
return true
|
||||
}
|
||||
return *n.IsEnabled
|
||||
}
|
||||
|
||||
// SetIsEnabled sets the IsEnabled field
|
||||
func (n *NotificationService) SetIsEnabled(value bool) {
|
||||
n.IsEnabled = &value
|
||||
}
|
||||
@@ -25,7 +25,7 @@ type TransferConfig struct {
|
||||
SourceShare string `form:"source_share"`
|
||||
SourceDomain string `form:"source_domain"`
|
||||
// FTP source fields
|
||||
SourcePassiveMode *bool `gorm:"default:true" form:"source_passive_mode"`
|
||||
SourcePassiveMode *bool `gorm:"default:true" form:"source_passive_mode"` // Already a pointer, no change needed here
|
||||
// OneDrive and Google Drive source fields
|
||||
SourceClientID string `form:"source_client_id"`
|
||||
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
||||
|
||||
Reference in new issue
Block a user