From b9acf47530d5b29c7583b558af3cf1f276ee0d25 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sat, 19 Apr 2025 13:22:40 -0700 Subject: [PATCH] Add gomftctl CLI tool and encryption key rotation utilities --- Dockerfile | 12 +- README.md | 53 +- cmd/gomftctl/main.go | 772 ++++++++++++++++++ docs/docs/advanced/admin-tools.md | 7 - docs/docs/advanced/command-line-tool.md | 278 +++++++ docs/sidebars.ts | 3 +- go.mod | 3 + go.sum | 8 + internal/config/config.go | 8 +- internal/db/migrate_provider_data.go | 181 +++- .../db/migrations/016_update_drive_type.go | 90 ++ internal/db/migrations/migrations.go | 1 + .../encryption/audit/key_rotation_utility.go | 50 +- internal/encryption/audit/key_rotator.go | 274 +++++++ .../encryption/keyrotation/key_rotator.go | 48 +- .../keyrotation/rotation_utility.go | 39 +- .../rotationmodel/rotation_types.go | 52 ++ 17 files changed, 1754 insertions(+), 125 deletions(-) create mode 100644 cmd/gomftctl/main.go create mode 100644 docs/docs/advanced/command-line-tool.md create mode 100644 internal/db/migrations/016_update_drive_type.go create mode 100644 internal/encryption/audit/key_rotator.go create mode 100644 internal/encryption/rotationmodel/rotation_types.go diff --git a/Dockerfile b/Dockerfile index d0a0242..6e8cd95 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,11 +69,16 @@ COPY . . # Generate template files from .templ files RUN templ generate -# Compile the application with version information +# Compile the main application with version information RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build \ -ldflags "-X github.com/starfleetcptn/gomft/components.AppVersion=${VERSION} -X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.Commit=${COMMIT} -X github.com/starfleetcptn/gomft/components.BuildTime=${BUILD_TIME} -X github.com/starfleetcptn/gomft/components.Commit=${COMMIT}" \ -o /app/gomft +# Compile the command line tool with version information +RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} go build \ + -ldflags "-X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.Commit=${COMMIT}" \ + -o /app/gomftctl ./cmd/gomftctl + # Install rclone with appropriate architecture RUN apk add --no-cache curl unzip && \ if [ "$TARGETARCH" = "arm64" ]; then \ @@ -112,8 +117,9 @@ RUN apk add --no-cache ca-certificates tzdata sqlite bash shadow su-exec \ RUN addgroup -g ${GID} ${USERNAME} && \ adduser -D -u ${UID} -G ${USERNAME} -s /bin/sh ${USERNAME} -# Copy the binary from the builder stage +# Copy the binaries from the builder stage COPY --from=builder /app/gomft /app/ +COPY --from=builder /app/gomftctl /app/ COPY --from=builder /usr/local/bin/rclone /usr/local/bin/rclone # Copy components @@ -130,7 +136,7 @@ RUN mkdir -p /app/data /app/backups RUN touch /app/.env && chmod 644 /app/.env && chown ${USERNAME}:${USERNAME} /app/.env # Set executable permissions -RUN chmod +x /app/gomft +RUN chmod +x /app/gomft /app/gomftctl # Set ownership of application files RUN chown -R ${USERNAME}:${USERNAME} /app diff --git a/README.md b/README.md index 4a551aa..1b0e464 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,20 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging

+## 📚 Documentation + +Comprehensive documentation is available at [https://starfleetcptn.github.io/GoMFT/](https://starfleetcptn.github.io/GoMFT/). + +The documentation includes: +- [Getting Started Guide](https://starfleetcptn.github.io/GoMFT/docs/getting-started) +- [Installation Instructions](https://starfleetcptn.github.io/GoMFT/docs/installation) +- [Configuration Reference](https://starfleetcptn.github.io/GoMFT/docs/configuration) +- [User Guide](https://starfleetcptn.github.io/GoMFT/docs/user-guide) +- [Storage Provider Setup](https://starfleetcptn.github.io/GoMFT/docs/storage-providers) +- [Advanced Features](https://starfleetcptn.github.io/GoMFT/docs/advanced) +- [Troubleshooting](https://starfleetcptn.github.io/GoMFT/docs/troubleshooting) +- [API Reference](https://starfleetcptn.github.io/GoMFT/docs/api) + > [!WARNING] > This application is actively under development. As such, any aspect of the application—including configurations, data structures, and database fields—may change rapidly and without prior notice. Please review all release notes thoroughly before updating. @@ -86,6 +100,7 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging - Optimized for both high-volume small files and large file transfers - Maximizes bandwidth utilization for cloud storage providers - **Web Interface**: User-friendly interface for managing transfers, built with Templ components +- **Command Line Tools**: Administrative tasks can be performed via the `gomftctl` CLI tool - **File Pattern Matching**: Support for file patterns to filter files during transfers - **File Output Patterns**: Dynamic naming of destination files using patterns with date variables - **Archive Function**: Option to archive transferred files for backup and compliance @@ -137,9 +152,10 @@ go install github.com/a-h/templ/cmd/templ@latest templ generate ``` -4. Build the application: +4. Build the application and CLI tools: ```bash go build -o gomft +go build -o gomftctl ./cmd/gomftctl ``` ### Docker Installation @@ -256,7 +272,40 @@ docker-compose up -d For more information and available tags, visit the [GoMFT Docker Hub page](https://hub.docker.com/r/starfleetcptn/gomft). -Full documentation is available at [https://starfleetcptn.github.io/GoMFT/](https://starfleetcptn.github.io/GoMFT/). +## Command Line Tools + +GoMFT includes a command line tool called `gomftctl` for administrative tasks: + +```bash +# View available commands +./gomftctl --help + +# Migrate provider data +./gomftctl migrate-providers + +# Rotate security keys +./gomftctl rotate-key --type jwt + +# Manage users +./gomftctl user create --email admin@example.com --password secure_password --admin +./gomftctl user list + +# Backup database +./gomftctl backup +``` + +See the [Admin Tools documentation](https://starfleetcptn.github.io/GoMFT/docs/advanced/admin-tools) for more details. + +## Contributing + +Contributions are welcome! Please see our [Contributing Guide](https://starfleetcptn.github.io/GoMFT/docs/contributing) for details on how to get started. + +1. Fork the repository +2. Create a feature branch +3. Make your changes +4. Submit a pull request + +We also welcome documentation improvements. The documentation source is available in the `docs/` directory. ## License diff --git a/cmd/gomftctl/main.go b/cmd/gomftctl/main.go new file mode 100644 index 0000000..f870a44 --- /dev/null +++ b/cmd/gomftctl/main.go @@ -0,0 +1,772 @@ +package main + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/starfleetcptn/gomft/internal/config" + "github.com/starfleetcptn/gomft/internal/db" + "github.com/starfleetcptn/gomft/internal/encryption" + "github.com/starfleetcptn/gomft/internal/encryption/keyrotation" + "golang.org/x/crypto/bcrypt" +) + +func main() { + // Create root command + rootCmd := &cobra.Command{ + Use: "gomftctl", + Short: "GoMFT Control Tool - Command line utilities for GoMFT", + Long: `GoMFT Control Tool (gomftctl) provides command line utilities for managing +your GoMFT installation, including database migrations, security key rotation, +and other administrative functions.`, + } + + // Add commands + rootCmd.AddCommand(createMigrateCmd()) + rootCmd.AddCommand(createKeyRotationCmd()) + rootCmd.AddCommand(createVersionCmd()) + rootCmd.AddCommand(createBackupCmd()) + rootCmd.AddCommand(createUserCmd()) + rootCmd.AddCommand(createEncryptionKeyRotationCmd()) + + // Execute the root command + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} + +// createMigrateCmd creates the migrate command for provider data migration +func createMigrateCmd() *cobra.Command { + var dryRun, validationOnly, force, debugMode, autoFill bool + var backupDir string + + migrateCmd := &cobra.Command{ + Use: "migrate-providers", + Short: "Migrate provider data to the new storage provider model", + Long: `Migrate provider data extracts unique provider configurations from existing +transfer configs and creates dedicated storage provider records. + +This command should be run when upgrading from older versions of GoMFT that +stored provider configuration directly in transfer configs.`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Set backup directory if not provided + if backupDir == "" { + backupDir = cfg.BackupDir + } + + // Initialize database + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + database, err := db.Initialize(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer database.Close() + + // Create migration options + options := db.MigrateProviderDataOptions{ + DryRun: dryRun, + ValidationOnly: validationOnly, + Force: force, + BackupDir: backupDir, + DebugMode: debugMode, + AutoFill: autoFill, + } + + // Run migration + fmt.Println("Starting provider data migration...") + stats, err := database.MigrateProviderData(options) + if err != nil { + fmt.Println("\nMigration failed with error:") + fmt.Printf("Error: %v\n", err) + + // Add more detailed error information + fmt.Println("\nDetailed error information:") + fmt.Println("===========================") + + // Unwrap nested errors if possible + var currentErr error = err + depth := 1 + for currentErr != nil { + fmt.Printf("%d. %v\n", depth, currentErr) + if unwrapped, ok := currentErr.(interface{ Unwrap() error }); ok { + currentErr = unwrapped.Unwrap() + depth++ + } else { + break + } + } + + // Print database connection information (without sensitive details) + fmt.Println("\nDatabase information:") + fmt.Printf("- Database path: %s\n", dbPath) + fmt.Printf("- Migration options: dryRun=%v, validationOnly=%v, force=%v\n", + options.DryRun, options.ValidationOnly, options.Force) + + log.Fatalf("Migration failed. See details above.") + } + + // Print report + fmt.Println(db.FormatMigrationReport(stats)) + }, + } + + // Add flags + migrateCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Simulate migration without making changes") + migrateCmd.Flags().BoolVar(&validationOnly, "validate-only", false, "Only validate if migration is possible without making changes") + migrateCmd.Flags().BoolVar(&force, "force", false, "Force migration even if validation fails") + migrateCmd.Flags().StringVar(&backupDir, "backup-dir", "", "Directory to store backup data (defaults to config backup_dir)") + migrateCmd.Flags().BoolVar(&debugMode, "debug", false, "Enable debug mode with more detailed error messages") + migrateCmd.Flags().BoolVar(&autoFill, "auto-fill", false, "Automatically fill missing required fields with placeholder values") + + return migrateCmd +} + +// createKeyRotationCmd creates the key rotation command +func createKeyRotationCmd() *cobra.Command { + var keyType string + var writeToEnv bool + + keyRotationCmd := &cobra.Command{ + Use: "rotate-key", + Short: "Rotate security keys used by GoMFT", + Long: `Rotate security keys generates new cryptographic keys for GoMFT. + +Available key types: +- jwt: JSON Web Token signing key +- totp: TOTP encryption key +- encryption: General encryption key used for sensitive data + +This command will generate a new key and provide instructions for updating +your configuration. The application must be restarted for changes to take effect.`, + Run: func(cmd *cobra.Command, args []string) { + // Validate key type + validTypes := map[string]string{ + "jwt": "JWT_SECRET", + "totp": "TOTP_ENCRYPTION_KEY", + "encryption": "GOMFT_ENCRYPTION_KEY", + } + + envVar, valid := validTypes[keyType] + if !valid { + log.Fatalf("Invalid key type: %s. Valid types are: jwt, totp, encryption", keyType) + } + + // Load configuration + _, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Generate a new key + newKey, err := generateSecureKey() + if err != nil { + log.Fatalf("Failed to generate secure key: %v", err) + } + + fmt.Printf("Generated new %s key: %s\n\n", keyType, newKey) + + if writeToEnv { + // Read current .env file + envPath := ".env" + envContent, err := os.ReadFile(envPath) + if err != nil { + log.Fatalf("Failed to read .env file: %v", err) + } + + // Update .env file with new key + updatedEnv, updated := updateEnvVar(string(envContent), envVar, newKey) + if !updated { + // If the variable wasn't found, append it + updatedEnv = updatedEnv + fmt.Sprintf("\n%s=%s\n", envVar, newKey) + } + + // Write updated content back to .env file + if err := os.WriteFile(envPath, []byte(updatedEnv), 0644); err != nil { + log.Fatalf("Failed to write updated .env file: %v", err) + } + + fmt.Printf("Updated %s in .env file\n", envVar) + fmt.Println("Please restart the GoMFT application for changes to take effect.") + } else { + // Print instructions for manual update + fmt.Println("To use this key, update your .env file with:") + fmt.Printf("%s=%s\n\n", envVar, newKey) + fmt.Println("Then restart the GoMFT application for changes to take effect.") + } + }, + } + + // Add flags + keyRotationCmd.Flags().StringVar(&keyType, "type", "", "Type of key to rotate (jwt, totp, encryption)") + keyRotationCmd.Flags().BoolVar(&writeToEnv, "write", false, "Write the new key directly to .env file") + keyRotationCmd.MarkFlagRequired("type") + + return keyRotationCmd +} + +// createVersionCmd creates the version command +func createVersionCmd() *cobra.Command { + return &cobra.Command{ + Use: "version", + Short: "Display version information", + Run: func(cmd *cobra.Command, args []string) { + // Import the version from the components package + fmt.Println("GoMFT Control Tool") + fmt.Println("Version: Same as GoMFT application") + fmt.Println("Visit https://github.com/starfleetcptn/gomft for more information") + }, + } +} + +// createBackupCmd creates the backup command +func createBackupCmd() *cobra.Command { + var outputDir string + + backupCmd := &cobra.Command{ + Use: "backup", + Short: "Create a backup of the GoMFT database", + Long: `Create a backup of the GoMFT database and configuration. +The backup includes the SQLite database file and the .env configuration file.`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Set output directory if not provided + if outputDir == "" { + outputDir = cfg.BackupDir + } + + // Ensure output directory exists + if err := os.MkdirAll(outputDir, 0755); err != nil { + log.Fatalf("Failed to create backup directory: %v", err) + } + + // Create timestamp for backup filename + timestamp := fmt.Sprintf("%s", filepath.Base(os.Args[0])) + + // Create backup + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + backupPath := filepath.Join(outputDir, fmt.Sprintf("gomft-backup-%s.db", timestamp)) + + // Copy database file + if err := copyFile(dbPath, backupPath); err != nil { + log.Fatalf("Failed to create database backup: %v", err) + } + + // Copy .env file if it exists + envPath := ".env" + backupEnvPath := filepath.Join(outputDir, fmt.Sprintf("gomft-env-backup-%s.env", timestamp)) + if _, err := os.Stat(envPath); err == nil { + if err := copyFile(envPath, backupEnvPath); err != nil { + log.Fatalf("Failed to backup .env file: %v", err) + } + fmt.Printf("Configuration backed up to: %s\n", backupEnvPath) + } + + fmt.Printf("Database backed up to: %s\n", backupPath) + }, + } + + // Add flags + backupCmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to store backup files (defaults to config backup_dir)") + + return backupCmd +} + +// createUserCmd creates the user management command +func createUserCmd() *cobra.Command { + userCmd := &cobra.Command{ + Use: "user", + Short: "User management commands", + Long: `Commands for managing GoMFT users, including creating, updating, and listing users.`, + } + + // Add subcommands + userCmd.AddCommand(createUserCreateCmd()) + userCmd.AddCommand(createUserResetPasswordCmd()) + userCmd.AddCommand(createUserListCmd()) + + return userCmd +} + +// createUserCreateCmd creates the user create command +func createUserCreateCmd() *cobra.Command { + var email, password string + var isAdmin bool + + createCmd := &cobra.Command{ + Use: "create", + Short: "Create a new user", + Long: `Create a new user with the specified email and password.`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Initialize database + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + database, err := db.Initialize(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer database.Close() + + // Create user by first generating password hash + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + log.Fatalf("Failed to hash password: %v", err) + } + + // Create user object + user := &db.User{ + Email: email, + PasswordHash: string(hashedPassword), + LastPasswordChange: time.Now(), + } + + // Set admin status if requested + if isAdmin { + user.SetIsAdmin(true) + } + + // Save user to database + if err := database.CreateUser(user); err != nil { + log.Fatalf("Failed to create user: %v", err) + } + + fmt.Printf("User created successfully:\n") + fmt.Printf(" ID: %d\n", user.ID) + fmt.Printf(" Email: %s\n", user.Email) + fmt.Printf(" Admin: %t\n", user.GetIsAdmin()) + }, + } + + // Add flags + createCmd.Flags().StringVar(&email, "email", "", "User email address") + createCmd.Flags().StringVar(&password, "password", "", "User password") + createCmd.Flags().BoolVar(&isAdmin, "admin", false, "Grant admin privileges to the user") + createCmd.MarkFlagRequired("email") + createCmd.MarkFlagRequired("password") + + return createCmd +} + +// createUserResetPasswordCmd creates the user reset-password command +func createUserResetPasswordCmd() *cobra.Command { + var email, newPassword string + + resetCmd := &cobra.Command{ + Use: "reset-password", + Short: "Reset a user's password", + Long: `Reset the password for a user with the specified email address.`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Initialize database + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + database, err := db.Initialize(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer database.Close() + + // Find user by email + var user db.User + if err := database.Where("email = ?", email).First(&user).Error; err != nil { + log.Fatalf("Failed to find user with email %s: %v", email, err) + } + + // Generate new password hash + hashedPassword, err := bcrypt.GenerateFromPassword([]byte(newPassword), bcrypt.DefaultCost) + if err != nil { + log.Fatalf("Failed to hash password: %v", err) + } + + // Update user password + user.PasswordHash = string(hashedPassword) + user.LastPasswordChange = time.Now() + + // Save user to database + if err := database.Save(&user).Error; err != nil { + log.Fatalf("Failed to update user: %v", err) + } + + fmt.Printf("Password reset successfully for user: %s\n", email) + }, + } + + // Add flags + resetCmd.Flags().StringVar(&email, "email", "", "User email address") + resetCmd.Flags().StringVar(&newPassword, "password", "", "New password") + resetCmd.MarkFlagRequired("email") + resetCmd.MarkFlagRequired("password") + + return resetCmd +} + +// createUserListCmd creates the user list command +func createUserListCmd() *cobra.Command { + return &cobra.Command{ + Use: "list", + Short: "List all users", + Long: `List all users in the GoMFT system.`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Initialize database + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + database, err := db.Initialize(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer database.Close() + + // Get all users + var users []db.User + if err := database.Find(&users).Error; err != nil { + log.Fatalf("Failed to get users: %v", err) + } + + // Print users + fmt.Println("GoMFT Users:") + fmt.Println("ID\tEmail\tAdmin\tLast Updated") + fmt.Println("--------------------------------------------------") + for _, user := range users { + lastUpdated := "Never" + if !user.UpdatedAt.IsZero() { + lastUpdated = user.UpdatedAt.Format("2006-01-02 15:04:05") + } + fmt.Printf("%d\t%s\t%t\t%s\n", user.ID, user.Email, user.GetIsAdmin(), lastUpdated) + } + }, + } +} + +// createEncryptionKeyRotationCmd creates the encryption key rotation command +func createEncryptionKeyRotationCmd() *cobra.Command { + var dryRun bool + var batchSize, maxErrors int + var backupDir string + var skipBackup bool + var oldKeyEnvVar string + var modelsFlag string + + rotateCmd := &cobra.Command{ + Use: "rotate-encryption-key", + Short: "Rotate encryption keys for sensitive data", + Long: `Rotate encryption keys for sensitive data stored in the database. + +This command will: +1. Create a backup of your database (unless --skip-backup is specified) +2. Re-encrypt all sensitive data with a new encryption key +3. Provide instructions for updating your configuration + +The application must be stopped before running this command to prevent data corruption. +`, + Run: func(cmd *cobra.Command, args []string) { + // Load configuration + cfg, err := config.Load() + if err != nil { + log.Fatalf("Failed to load configuration: %v", err) + } + + // Set backup directory if not provided + if backupDir == "" { + backupDir = cfg.BackupDir + } + + // Create backup if needed + if !skipBackup { + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + backupPath := filepath.Join(backupDir, fmt.Sprintf("gomft_backup_before_key_rotation_%s.db", + time.Now().Format("20060102_150405"))) + + fmt.Printf("Creating database backup at %s...\n", backupPath) + if err := copyFile(dbPath, backupPath); err != nil { + log.Fatalf("Failed to create backup: %v", err) + } + fmt.Println("Backup created successfully.") + } else { + fmt.Println("Skipping database backup as requested.") + } + + // Initialize database + dbPath := filepath.Join(cfg.DataDir, "gomft.db") + database, err := db.Initialize(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer database.Close() + + // Setup old encryption service + if oldKeyEnvVar == "" { + oldKeyEnvVar = encryption.DefaultKeyEnvVar + } + + // Get the current encryption service + oldService, err := encryption.GetGlobalEncryptionService() + if err != nil { + log.Fatalf("Failed to get current encryption service: %v", err) + } + + // Generate new key + newKey, err := encryption.GenerateKey(encryption.AES256KeySize) + if err != nil { + log.Fatalf("Failed to generate new encryption key: %v", err) + } + + // Create new key manager for the new key + newKeyManager := &keyManager{key: newKey} + + // Setup new encryption service with the new key + newService, err := encryption.NewEncryptionService(newKeyManager) + if err != nil { + log.Fatalf("Failed to create new encryption service: %v", err) + } + + // Create rotation options + options := keyrotation.RotationOptions{ + DryRun: dryRun, + BatchSize: batchSize, + MaxErrors: maxErrors, + Timeout: 24 * time.Hour, + ProgressCallback: func(modelName string, processed, total int) { + fmt.Printf("\rProcessing %s: %d/%d records (%.1f%%)", + modelName, processed, total, float64(processed)/float64(total)*100) + }, + } + + // Create rotation utility + rotationUtil, err := keyrotation.NewRotationUtility( + database.DB, // Use the underlying gorm.DB + oldService, + newService, + nil, // No auditor needed, keyrotation will use the global one + options, + ) + if err != nil { + log.Fatalf("Failed to create rotation utility: %v", err) + } + + // Find models with encrypted fields + var models []interface{} + if modelsFlag == "auto" { + fmt.Println("Automatically detecting models with encrypted fields...") + models, err = rotationUtil.FindModelsWithEncryptedFields() + if err != nil { + log.Fatalf("Failed to find models with encrypted fields: %v", err) + } + if len(models) == 0 { + log.Fatalf("No models with encrypted fields found") + } + } else if modelsFlag != "" { + // TODO: Support manual model specification + log.Fatalf("Manual model specification not yet implemented, use --models=auto") + } else { + log.Fatalf("No models specified, use --models=auto to automatically detect models") + } + + // Create migration plan + fmt.Println("Creating encryption migration plan...") + plan, err := rotationUtil.CreateEncryptionMigrationPlan(models) + if err != nil { + log.Fatalf("Failed to create migration plan: %v", err) + } + + // Print plan + fmt.Println("\nEncryption Migration Plan:") + fmt.Printf("Total models: %d\n", len(plan.ModelPlans)) + fmt.Printf("Total records: %d\n", plan.EstimatedRecords) + fmt.Printf("Estimated duration: %s\n", plan.EstimatedDuration.Round(time.Second)) + fmt.Println("\nModels to process:") + for name, modelPlan := range plan.ModelPlans { + fmt.Printf("- %s: %d records, %d encrypted fields\n", + name, modelPlan.RecordCount, len(modelPlan.EncryptedFields)) + } + + // Confirm if not in dry run mode + if !dryRun { + fmt.Println("\nWARNING: This operation will re-encrypt all sensitive data with a new key.") + fmt.Println("Make sure the application is stopped before proceeding.") + fmt.Print("\nDo you want to continue? [y/N]: ") + var response string + fmt.Scanln(&response) + if strings.ToLower(response) != "y" { + fmt.Println("Operation cancelled.") + return + } + } + + // Perform key rotation + fmt.Println("\nStarting key rotation...") + startTime := time.Now() + stats, err := rotationUtil.RotateKeysForModels(context.Background(), models) + if err != nil { + fmt.Println("\nKey rotation failed with error:") + fmt.Printf("Error: %v\n", err) + + // Add more detailed error information + fmt.Println("\nDetailed error information:") + fmt.Println("===========================") + + // Unwrap nested errors if possible + var currentErr error = err + depth := 1 + for currentErr != nil { + fmt.Printf("%d. %v\n", depth, currentErr) + if unwrapped, ok := currentErr.(interface{ Unwrap() error }); ok { + currentErr = unwrapped.Unwrap() + depth++ + } else { + break + } + } + + // Print rotation configuration details + fmt.Println("\nRotation configuration:") + fmt.Printf("- Dry run: %v\n", dryRun) + fmt.Printf("- Batch size: %d\n", batchSize) + fmt.Printf("- Max errors: %d\n", maxErrors) + fmt.Printf("- Models: %s\n", modelsFlag) + fmt.Printf("- Old key env var: %s\n", oldKeyEnvVar) + + log.Fatalf("Key rotation failed. See details above.") + } + duration := time.Since(startTime).Round(time.Second) + + // Print results + fmt.Println("\nKey rotation completed successfully!") + fmt.Printf("Total records processed: %d/%d\n", stats.ProcessedRecords, stats.TotalRecords) + fmt.Printf("Failed records: %d\n", stats.FailedRecords) + fmt.Printf("Duration: %s\n", duration) + + if len(stats.Errors) > 0 { + fmt.Printf("\nErrors (%d):\n", len(stats.Errors)) + for i, err := range stats.Errors { + if i >= 10 { + fmt.Printf("... and %d more errors\n", len(stats.Errors)-10) + break + } + fmt.Printf("- %s\n", err) + } + } + + // Print next steps + if !dryRun { + fmt.Println("\nNext steps:") + fmt.Println("1. Update your environment variable or .env file with the new encryption key:") + fmt.Printf(" %s=%s\n", oldKeyEnvVar, base64.StdEncoding.EncodeToString(newKey)) + fmt.Println("2. Restart your GoMFT application") + fmt.Println("\nIMPORTANT: Keep a backup of both the old and new keys until you verify everything works correctly.") + } else { + fmt.Println("\nDry run completed. No changes were made to the database.") + fmt.Println("Run without --dry-run to perform the actual key rotation.") + } + }, + } + + // Add flags + rotateCmd.Flags().BoolVar(&dryRun, "dry-run", false, "Simulate key rotation without making changes") + rotateCmd.Flags().IntVar(&batchSize, "batch-size", 100, "Number of records to process in each batch") + rotateCmd.Flags().IntVar(&maxErrors, "max-errors", 50, "Maximum number of errors before aborting") + rotateCmd.Flags().StringVar(&backupDir, "backup-dir", "", "Directory to store backup data (defaults to config backup_dir)") + rotateCmd.Flags().BoolVar(&skipBackup, "skip-backup", false, "Skip database backup (not recommended)") + rotateCmd.Flags().StringVar(&oldKeyEnvVar, "old-key-env", "", "Environment variable containing the old encryption key (defaults to GOMFT_ENCRYPTION_KEY)") + rotateCmd.Flags().StringVar(&modelsFlag, "models", "auto", "Models to process (use 'auto' for automatic detection)") + + return rotateCmd +} + +// keyManager is a simple implementation of the encryption.KeyManager interface +// that uses a fixed key for the new encryption service +type keyManager struct { + key []byte +} + +func (km *keyManager) Initialize() error { + // Already initialized with the key + return nil +} + +func (km *keyManager) GetPrimaryKey() ([]byte, error) { + return km.key, nil +} + +func (km *keyManager) GetEnvironmentVariableName() string { + return "TEMP_KEY_MANAGER" +} + +func (km *keyManager) StoreKeyEnvironment(key []byte) error { + // Not needed for this implementation + return nil +} + +// Helper functions + +// generateSecureKey creates a cryptographically secure random key encoded as base64 +func generateSecureKey() (string, error) { + return config.GenerateSecureKey() +} + +// updateEnvVar updates an environment variable in the .env file content +func updateEnvVar(content, key, value string) (string, bool) { + lines := strings.Split(content, "\n") + prefix := key + "=" + updated := false + + for i, line := range lines { + if strings.HasPrefix(line, prefix) { + lines[i] = prefix + value + updated = true + break + } + } + + return strings.Join(lines, "\n"), updated +} + +// copyFile copies a file from src to dst +func copyFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + dstFile, err := os.Create(dst) + if err != nil { + return err + } + defer dstFile.Close() + + _, err = io.Copy(dstFile, srcFile) + return err +} diff --git a/docs/docs/advanced/admin-tools.md b/docs/docs/advanced/admin-tools.md index 8883c99..90dd1dc 100644 --- a/docs/docs/advanced/admin-tools.md +++ b/docs/docs/advanced/admin-tools.md @@ -7,13 +7,6 @@ title: Admin Tools GoMFT provides a comprehensive set of administrative tools for system management, monitoring, and maintenance. These tools help administrators maintain the system, troubleshoot issues, and ensure optimal performance. -## Accessing Admin Tools - -Admin tools are available to users with administrator privileges: - -1. Log in with an administrator account -2. Navigate to **Admin Tools** in the sidebar menu - ## Log Viewer The Admin Tools panel includes an integrated log viewer with the following features: diff --git a/docs/docs/advanced/command-line-tool.md b/docs/docs/advanced/command-line-tool.md new file mode 100644 index 0000000..eff7a3e --- /dev/null +++ b/docs/docs/advanced/command-line-tool.md @@ -0,0 +1,278 @@ +--- +sidebar_position: 8 +title: Command Line Tools +--- + +GoMFT provides a command line tool called `gomftctl` that allows administrators to perform various management tasks without using the web interface. This tool is particularly useful for automation, scripting, and performing administrative tasks in environments where the web UI is not accessible. + +## Installation + +The `gomftctl` tool is included with your GoMFT installation. You can find it in the root directory of your GoMFT installation. + +If you need to build it manually, you can do so with: + +```bash +cd /path/to/gomft +go build -o gomftctl ./cmd/gomftctl +``` + +### Using with Docker + +If you're running GoMFT in a Docker container, the `gomftctl` tool is already included in the container. You can run it using the `docker exec` command: + +```bash +# Replace gomft-container with your actual container name +docker exec -it gomft-container /app/gomftctl [command] [options] +``` + +For example, to view the version information: + +```bash +docker exec -it gomft-container /app/gomftctl version +``` + +For commands that require stopping the application first (like key rotation), you'll need to: + +1. Stop the container +2. Run the command in a new container using the same volumes +3. Restart the original container + +```bash +# Stop the container +docker stop gomft-container + +# Run a command using the same volumes +docker run --rm -v gomft_data:/app/data -v gomft_backups:/app/backups gomft/gomft:latest /app/gomftctl [command] [options] + +# Restart the container +docker start gomft-container +``` + +## Available Commands + +The `gomftctl` tool provides the following commands: + +### Provider Data Migration + +Migrate provider data from older versions of GoMFT to the new storage provider model: + +```bash +./gomftctl migrate-providers [--dry-run] [--validate-only] [--force] [--backup-dir PATH] [--debug] [--auto-fill] +``` + +Options: +- `--dry-run`: Simulate migration without making changes +- `--validate-only`: Only validate if migration is possible without making changes +- `--force`: Force migration even if validation fails +- `--backup-dir`: Directory to store backup data (defaults to config backup_dir) +- `--debug`: Enable debug mode with more detailed error messages +- `--auto-fill`: Automatically fill missing required fields with placeholder values + +### Security Key Rotation + +Generate new security keys for the application: + +```bash +./gomftctl rotate-key --type [jwt|totp|encryption] [--write] +``` + +Options: +- `--type`: Type of key to rotate (required) + - `jwt`: JSON Web Token signing key + - `totp`: TOTP encryption key + - `encryption`: General encryption key used for sensitive data +- `--write`: Write the new key directly to .env file (otherwise just displays the key) + +### Encryption Key Rotation + +Rotate encryption keys for sensitive data stored in the database: + +```bash +./gomftctl rotate-encryption-key [--dry-run] [--batch-size SIZE] [--max-errors NUM] [--backup-dir PATH] [--skip-backup] [--old-key-env VAR] [--models MODE] +``` + +This command will: +1. Create a backup of your database (unless `--skip-backup` is specified) +2. Re-encrypt all sensitive data with a new encryption key +3. Provide instructions for updating your configuration + +**Important**: The application must be stopped before running this command to prevent data corruption. + +Options: +- `--dry-run`: Simulate key rotation without making changes +- `--batch-size`: Number of records to process in each batch (default 100) +- `--max-errors`: Maximum number of errors before aborting (default 50) +- `--backup-dir`: Directory to store backup data (defaults to config backup_dir) +- `--skip-backup`: Skip database backup (not recommended) +- `--old-key-env`: Environment variable containing the old encryption key (defaults to GOMFT_ENCRYPTION_KEY) +- `--models`: Models to process (use 'auto' for automatic detection, default 'auto') + +### Database Backup + +Create a backup of the GoMFT database and configuration: + +```bash +./gomftctl backup [--output-dir PATH] +``` + +Options: +- `--output-dir`: Directory to store backup files (defaults to config backup_dir) + +### User Management + +Commands for managing GoMFT users: + +#### Create a new user + +```bash +./gomftctl user create --email EMAIL --password PASSWORD [--admin] +``` + +Options: +- `--email`: User email address (required) +- `--password`: User password (required) +- `--admin`: Grant admin privileges to the user + +#### Reset a user's password + +```bash +./gomftctl user reset-password --email EMAIL --password PASSWORD +``` + +Options: +- `--email`: User email address (required) +- `--password`: New password (required) + +#### List all users + +```bash +./gomftctl user list +``` + +### Version Information + +Display version information: + +```bash +./gomftctl version +``` + +## Examples + +### Migrating Provider Data + +To migrate provider data with a dry run first: + +```bash +# First do a dry run to see what would happen +./gomftctl migrate-providers --dry-run + +# Then run the actual migration +./gomftctl migrate-providers +``` + +If you encounter errors due to missing required fields, you can use the auto-fill option: + +```bash +# Migrate with auto-fill to handle missing required fields +./gomftctl migrate-providers --auto-fill + +# For more detailed error information, add the debug flag +./gomftctl migrate-providers --auto-fill --debug +``` + +When using `--auto-fill`, the system will: +1. Automatically supply placeholder values for missing required fields +2. Mark providers with "[AUTO-FILLED]" in their names +3. Log warnings about which fields were auto-filled +4. Allow you to update the correct values after migration + +### Rotating JWT Secret Key + +To rotate the JWT secret key and update the .env file: + +```bash +./gomftctl rotate-key --type jwt --write +``` + +### Rotating Encryption Key for Sensitive Data + +To rotate the encryption key used for sensitive data in the database: + +```bash +# First stop the GoMFT application +systemctl stop gomft + +# Run a dry run to see what would be affected +./gomftctl rotate-encryption-key --dry-run + +# Perform the actual key rotation +./gomftctl rotate-encryption-key + +# Update your environment variable or .env file with the new key +# Then restart the application +systemctl start gomft +``` + +#### With Docker + +To rotate encryption keys when running GoMFT in Docker: + +```bash +# Stop the container +docker stop gomft-container + +# Run a dry run to see what would be affected +docker run --rm -v gomft_data:/app/data -v gomft_backups:/app/backups gomft/gomft:latest /app/gomftctl rotate-encryption-key --dry-run + +# Perform the actual key rotation +docker run --rm -v gomft_data:/app/data -v gomft_backups:/app/backups gomft/gomft:latest /app/gomftctl rotate-encryption-key + +# Update your environment variables in your docker-compose.yml or run command +# Then restart the container +docker start gomft-container +``` + +### Creating an Admin User + +To create a new admin user: + +```bash +./gomftctl user create --email admin@example.com --password secure_password --admin +``` + +### Backing Up the Database + +To create a backup of the database: + +```bash +./gomftctl backup --output-dir /path/to/backup/directory +``` + +## Using in Scripts + +The `gomftctl` tool is designed to be used in scripts and automation. For example, you could create a cron job to backup the database daily: + +```bash +# Add to crontab +0 2 * * * /path/to/gomft/gomftctl backup --output-dir /path/to/backup/directory +``` + +Or you could create a script to rotate security keys periodically: + +```bash +#!/bin/bash +# Stop the GoMFT service +systemctl stop gomft + +# Rotate all security keys +/path/to/gomft/gomftctl rotate-key --type jwt --write +/path/to/gomft/gomftctl rotate-key --type totp --write +/path/to/gomft/gomftctl rotate-key --type encryption --write + +# Rotate encryption key for sensitive data in the database +/path/to/gomft/gomftctl rotate-encryption-key + +# Restart the GoMFT service to apply changes +systemctl restart gomft +``` diff --git a/docs/sidebars.ts b/docs/sidebars.ts index 2880e2c..ec0c712 100644 --- a/docs/sidebars.ts +++ b/docs/sidebars.ts @@ -38,13 +38,14 @@ const sidebars: SidebarsConfig = { type: 'category', label: 'Advanced Features', items: [ + 'advanced/admin-tools', + 'advanced/command-line-tool', 'advanced/notifications-overview', 'advanced/gotify-notifications', 'advanced/ntfy-notifications', 'advanced/pushbullet-notifications', 'advanced/pushover-notifications', 'advanced/webhook-notifications', - 'advanced/admin-tools' ], }, { diff --git a/go.mod b/go.mod index da24216..004a4a3 100644 --- a/go.mod +++ b/go.mod @@ -37,6 +37,7 @@ require ( github.com/gorilla/context v1.1.2 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/gorilla/sessions v1.4.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -48,6 +49,8 @@ require ( github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/spf13/cobra v1.9.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect diff --git a/go.sum b/go.sum index ba653f0..6500a87 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,7 @@ github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFos github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -58,6 +59,8 @@ github.com/gorilla/sessions v1.4.0 h1:kpIYOp/oi6MG/p5PgxApU8srsSw9tuFbt46Lt7auzq github.com/gorilla/sessions v1.4.0/go.mod h1:FLWm50oby91+hl7p/wRxDth9bWSuk0qVL2emc7lT5ik= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -90,6 +93,11 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo= +github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= diff --git a/internal/config/config.go b/internal/config/config.go index 45e7a02..e0f6cfe 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,12 +37,12 @@ type EmailConfig struct { func Load() (*Config, error) { // Generate secure encryption keys - defaultTOTPKey, err := generateSecureKey() + defaultTOTPKey, err := GenerateSecureKey() if err != nil { return nil, err } - defaultGOMFTKey, err := generateSecureKey() + defaultGOMFTKey, err := GenerateSecureKey() if err != nil { return nil, err } @@ -200,8 +200,8 @@ func Load() (*Config, error) { return cfg, nil } -// generateSecureKey creates a cryptographically secure random key encoded as base64 -func generateSecureKey() (string, error) { +// GenerateSecureKey creates a cryptographically secure random key encoded as base64 +func GenerateSecureKey() (string, error) { // Generate 32 bytes of random data (256 bits) bytes := make([]byte, 32) _, err := rand.Read(bytes) diff --git a/internal/db/migrate_provider_data.go b/internal/db/migrate_provider_data.go index 0afcd12..0b496e0 100644 --- a/internal/db/migrate_provider_data.go +++ b/internal/db/migrate_provider_data.go @@ -1,12 +1,15 @@ package db import ( + "context" "fmt" "log" + "regexp" "strings" "time" "github.com/starfleetcptn/gomft/internal/encryption" + "gorm.io/gorm" ) // ProviderConfig represents a unique provider configuration extracted from TransferConfigs @@ -134,6 +137,8 @@ type MigrateProviderDataOptions struct { ValidationOnly bool // If true, only perform validation without migration Force bool // If true, ignore validation errors and proceed with migration BackupDir string // Directory to store backups in + DebugMode bool // If true, provide more detailed error information + AutoFill bool // If true, automatically fill missing required fields with placeholder values } // ExtractUniqueProviderConfigs extracts all unique provider configurations from existing TransferConfig records @@ -399,11 +404,56 @@ func (db *DB) CreateStorageProviderRecords(uniqueConfigs map[string]*ProviderCon provider.EncryptedClientSecret = encryptedClientSecret } + // Check if we should attempt to auto-fill missing required fields + autoFill := false + if tx.Statement != nil && tx.Statement.Context != nil { + if autoFillVal := tx.Statement.Context.Value("auto_fill"); autoFillVal != nil { + if af, ok := autoFillVal.(bool); ok { + autoFill = af + } + } + } + // Create the provider record if err := tx.Create(provider).Error; err != nil { - sanitizedErrMsg := sanitizeErrorMessage(err.Error()) + // Check if we should try to auto-fill missing fields + if autoFill { + // Try to determine if it's a missing required field error + errStr := err.Error() + if strings.Contains(strings.ToLower(errStr), "not null") || + strings.Contains(strings.ToLower(errStr), "required") || + strings.Contains(strings.ToLower(errStr), "cannot be null") { + + // Try to auto-fill missing fields based on provider type + filled := autoFillMissingFields(provider, providerConfig) + if filled { + // Try again with auto-filled fields + if retryErr := tx.Create(provider).Error; retryErr == nil { + // Success! Add a warning to the log + log.Printf("WARNING: Auto-filled missing required fields for provider %s. Please update this provider with correct values.", provider.Name) + // Continue with normal flow + goto successLabel + } else { + // Still failed, proceed with normal error handling + err = retryErr + } + } + } + } + + // Get debug mode flag from context + debugMode := false + if tx.Statement != nil && tx.Statement.Context != nil { + if debugVal := tx.Statement.Context.Value("debug_mode"); debugVal != nil { + if debug, ok := debugVal.(bool); ok { + debugMode = debug + } + } + } + sanitizedErrMsg := sanitizeErrorMessage(err.Error(), debugMode) return rollback(fmt.Errorf("failed to create provider record: %s", sanitizedErrMsg)) } + successLabel: // Store the provider ID in the map providerIDMap[key] = provider.ID @@ -424,8 +474,127 @@ func (db *DB) CreateStorageProviderRecords(uniqueConfigs map[string]*ProviderCon return providerIDMap, nil } +// autoFillMissingFields attempts to fill missing required fields with placeholder values +// Returns true if fields were filled, false otherwise +func autoFillMissingFields(provider *StorageProvider, config *ProviderConfig) bool { + changesMade := false + + // Check and fill common fields that might be required + if provider.Name == "" { + provider.Name = fmt.Sprintf("Auto-filled Provider %s", time.Now().Format("2006-01-02 15:04:05")) + changesMade = true + } + + // Fill fields based on provider type + switch provider.Type { + case ProviderTypeSFTP, ProviderTypeFTP, ProviderTypeHetzner: + // Fill host if empty + if provider.Host == "" { + provider.Host = "placeholder.example.com" + changesMade = true + } + // Fill port if zero + if provider.Port == 0 { + if provider.Type == ProviderTypeFTP { + provider.Port = 21 + } else { + provider.Port = 22 // Default SFTP port + } + changesMade = true + } + // Fill username if empty + if provider.Username == "" { + provider.Username = "placeholder_user" + changesMade = true + } + + case ProviderTypeS3: + // Fill bucket if empty + if provider.Bucket == "" { + provider.Bucket = "placeholder-bucket" + changesMade = true + } + // Fill region if empty + if provider.Region == "" { + provider.Region = "us-east-1" + changesMade = true + } + // Fill access key if empty + if provider.AccessKey == "" { + provider.AccessKey = "PLACEHOLDER_ACCESS_KEY" + changesMade = true + } + // Fill endpoint if empty + if provider.Endpoint == "" { + provider.Endpoint = "https://s3.amazonaws.com" + changesMade = true + } + + case ProviderTypeSMB: + // Fill host if empty + if provider.Host == "" { + provider.Host = "placeholder-smb-server" + changesMade = true + } + // Fill share if empty + if provider.Share == "" { + provider.Share = "placeholder-share" + changesMade = true + } + // Fill domain if empty + if provider.Domain == "" { + provider.Domain = "WORKGROUP" + changesMade = true + } + + case ProviderTypeOneDrive, ProviderTypeGoogleDrive, ProviderTypeGooglePhoto: + // Fill client ID if empty + if provider.ClientID == "" { + provider.ClientID = "placeholder-client-id" + changesMade = true + } + // Fill drive ID if empty for OneDrive/Google Drive + if provider.DriveID == "" && (provider.Type == ProviderTypeOneDrive || provider.Type == ProviderTypeGoogleDrive) { + provider.DriveID = "placeholder-drive-id" + changesMade = true + } + } + + // Set authenticated to false by default if it's nil + if provider.Authenticated == nil { + falseVal := false + provider.Authenticated = &falseVal + changesMade = true + } + + // If we made changes, log a warning + if changesMade { + log.Printf("WARNING: Auto-filled missing required fields for provider type %s. Please update with correct values.", provider.Type) + } + + return changesMade +} + // sanitizeErrorMessage removes any potential sensitive information from error messages -func sanitizeErrorMessage(errMsg string) string { +func sanitizeErrorMessage(errMsg string, debugMode bool) string { + // In debug mode, we'll provide more information but still sanitize critical parts + if debugMode { + // List of sensitive keywords to check for and redact + sensitiveKeywords := []string{ + "password", "secret", "token", "key", "credential", "auth", + } + + // Redact sensitive information with placeholders instead of hiding the whole message + sanitizedMsg := errMsg + for _, keyword := range sensitiveKeywords { + // Case insensitive replacement using regex + re := regexp.MustCompile(fmt.Sprintf(`(?i)(%s\s*[:=]\s*['"]*)[^'"\s]+(['"]*\s*)`, keyword)) + sanitizedMsg = re.ReplaceAllString(sanitizedMsg, "${1}[REDACTED]${2}") + } + return fmt.Sprintf("DEBUG: %s", sanitizedMsg) + } + + // Standard non-debug mode with strict security // List of sensitive keywords to check for sensitiveKeywords := []string{ "password", "secret", "token", "key", "credential", "auth", @@ -719,6 +888,14 @@ func (db *DB) ValidateMigrationIntegrity() (*ValidationResult, error) { // MigrateProviderData is the main function that performs the complete migration process func (db *DB) MigrateProviderData(options MigrateProviderDataOptions) (*MigrationStats, error) { + // Store options in context for use by other functions + ctx := context.Background() + ctx = context.WithValue(ctx, "debug_mode", options.DebugMode) + ctx = context.WithValue(ctx, "auto_fill", options.AutoFill) + // Create a new DB session with the context + dbWithContext := db.DB.WithContext(ctx).Session(&gorm.Session{}) + // Update our DB wrapper to use this session + db.DB = dbWithContext // Initialize migration stats stats := &MigrationStats{ StartTime: time.Now(), diff --git a/internal/db/migrations/016_update_drive_type.go b/internal/db/migrations/016_update_drive_type.go new file mode 100644 index 0000000..e3c6b58 --- /dev/null +++ b/internal/db/migrations/016_update_drive_type.go @@ -0,0 +1,90 @@ +package migrations + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// UpdateDriveType updates the source_type and dest_type from 'gdrive' to 'drive' +func UpdateDriveType() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "016_update_drive_type", + Migrate: func(tx *gorm.DB) error { + // Check if any tables exist (indicating an existing database) + var count int64 + if err := tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'").Scan(&count).Error; err != nil { + return fmt.Errorf("failed to check for existing tables: %v", err) + } + + // If tables exist, create a backup + if count > 0 { + // Get the database path + sqlDB, err := tx.DB() + if err != nil { + return fmt.Errorf("failed to get underlying database: %v", err) + } + + var seq int + var name, dbPath string + if err := sqlDB.QueryRow("PRAGMA database_list").Scan(&seq, &name, &dbPath); err != nil { + return fmt.Errorf("failed to get database path: %v", err) + } + + // Get backup directory from environment variable or use default + backupDir := os.Getenv("BACKUP_DIR") + if backupDir == "" { + backupDir = "/app/backups" // Default Docker path + // Check if we're not in Docker + if _, err := os.Stat(backupDir); os.IsNotExist(err) { + backupDir = "backups" // Fallback to local directory + } + } + + // Create backup directory if it doesn't exist + if err := os.MkdirAll(backupDir, 0755); err != nil { + return fmt.Errorf("failed to create backup directory: %v", err) + } + + // Create backup file with timestamp in the backup directory + dbFileName := filepath.Base(dbPath) + backupFileName := fmt.Sprintf("%s.backup.%s", dbFileName, time.Now().Format("20060102_150405")) + backupFile := filepath.Join(backupDir, backupFileName) + + // Read original database + data, err := os.ReadFile(dbPath) + if err != nil { + return fmt.Errorf("failed to read database for backup: %v", err) + } + + // Write backup + if err := os.WriteFile(backupFile, data, 0600); err != nil { + return fmt.Errorf("failed to create database backup: %v", err) + } + + fmt.Printf("Created database backup at: %s\n", backupFile) + } + + // Update source_type + if err := tx.Exec(`UPDATE transfer_configs SET source_type = 'drive' WHERE source_type = 'gdrive'`).Error; err != nil { + return err + } + + // Update dest_type + return tx.Exec(`UPDATE transfer_configs SET dest_type = 'drive' WHERE dest_type = 'gdrive'`).Error + }, + Rollback: func(tx *gorm.DB) error { + // Revert source_type + if err := tx.Exec(`UPDATE transfer_configs SET source_type = 'gdrive' WHERE source_type = 'drive'`).Error; err != nil { + return err + } + + // Revert dest_type + return tx.Exec(`UPDATE transfer_configs SET dest_type = 'gdrive' WHERE dest_type = 'drive'`).Error + }, + } +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 6f17812..c5e3939 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -29,6 +29,7 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate { CleanupInvalidBooleans(), // 013 AddStorageProviders(), // 014 AddProviderRefsToTransferConfig(), // 015 + UpdateDriveType(), // 016 ) return gormigrate.New(db, gormigrate.DefaultOptions, migrations) diff --git a/internal/encryption/audit/key_rotation_utility.go b/internal/encryption/audit/key_rotation_utility.go index d308e27..e270a6d 100644 --- a/internal/encryption/audit/key_rotation_utility.go +++ b/internal/encryption/audit/key_rotation_utility.go @@ -10,27 +10,12 @@ import ( "time" "github.com/starfleetcptn/gomft/internal/encryption" - "github.com/starfleetcptn/gomft/internal/encryption/keyrotation" + "github.com/starfleetcptn/gomft/internal/encryption/rotationmodel" "gorm.io/gorm" ) -// RotationOptions contains configuration for the key rotation process -type RotationOptions struct { - // DryRun performs all operations but doesn't save changes to database - DryRun bool - // BatchSize sets the number of records to process in each batch - BatchSize int - // MaxErrors sets the threshold of errors before aborting - MaxErrors int - // Parallelism controls how many models are processed in parallel - Parallelism int - // Timeout specifies a maximum duration for the entire operation - Timeout time.Duration - // WorkerTimeout specifies maximum duration for a single batch - WorkerTimeout time.Duration - // ProgressCallback receives updates on rotation progress - ProgressCallback func(modelName string, processed, total int) -} +// For backward compatibility +type RotationOptions = rotationmodel.RotationOptions // RotationUtility provides comprehensive capabilities for rotating encryption keys // across multiple database models with detailed auditing and progress tracking @@ -125,19 +110,19 @@ func (r *RotationUtility) runHook(name string, data interface{}) error { } // RotateKeysForModels performs key rotation for multiple model types with detailed monitoring -func (r *RotationUtility) RotateKeysForModels(ctx context.Context, models []interface{}) (*keyrotation.RotationStats, error) { +func (r *RotationUtility) RotateKeysForModels(ctx context.Context, models []interface{}) (*rotationmodel.RotationStats, error) { // Create master context with timeout masterCtx, cancel := context.WithTimeout(ctx, r.options.Timeout) defer cancel() // Track overall stats - overallStats := &keyrotation.RotationStats{ + overallStats := &rotationmodel.RotationStats{ StartTime: time.Now(), Errors: make([]string, 0), } - // Create key rotator - rotator, err := keyrotation.NewKeyRotator(r.db, r.oldService, r.newService, r.auditor) + // Create key rotator - we'll implement our own version instead of using keyrotation package + rotator, err := NewKeyRotator(r.db, r.oldService, r.newService, r.auditor) if err != nil { return overallStats, fmt.Errorf("failed to create key rotator: %w", err) } @@ -194,7 +179,7 @@ func (r *RotationUtility) RotateKeysForModels(ctx context.Context, models []inte // Create a goroutine to handle timeouts rotationDone := make(chan struct{}) - var modelStats *keyrotation.RotationStats + var modelStats *rotationmodel.RotationStats var rotationErr error go func() { @@ -494,19 +479,8 @@ func (r *RotationUtility) estimateMigrationTime(recordCount, fieldCount int) tim return time.Duration(totalTimeMs) * time.Millisecond } -// EncryptionMigrationPlan contains the complete plan for migration -type EncryptionMigrationPlan struct { - ModelPlans map[string]*ModelMigrationPlan `json:"model_plans"` - EstimatedDuration time.Duration `json:"estimated_duration"` - EstimatedRecords int `json:"estimated_records"` - RecommendedOptions RotationOptions `json:"recommended_options"` -} +// For backward compatibility +type EncryptionMigrationPlan = rotationmodel.EncryptionMigrationPlan -// ModelMigrationPlan contains migration details for a specific model -type ModelMigrationPlan struct { - ModelName string `json:"model_name"` - RecordCount int `json:"record_count"` - EstimatedTime time.Duration `json:"estimated_time"` - EncryptedFields []string `json:"encrypted_fields"` - BatchSizeRec int `json:"batch_size_recommendation"` -} +// For backward compatibility +type ModelMigrationPlan = rotationmodel.ModelMigrationPlan diff --git a/internal/encryption/audit/key_rotator.go b/internal/encryption/audit/key_rotator.go new file mode 100644 index 0000000..8acab0a --- /dev/null +++ b/internal/encryption/audit/key_rotator.go @@ -0,0 +1,274 @@ +package audit + +import ( + "errors" + "fmt" + "reflect" + "strings" + "time" + + "github.com/starfleetcptn/gomft/internal/encryption" + "github.com/starfleetcptn/gomft/internal/encryption/rotationmodel" + "gorm.io/gorm" +) + +// Common errors +var ( + ErrNoOldKey = errors.New("old encryption key not found") + ErrNoNewKey = errors.New("new encryption key not found") + ErrSameKey = errors.New("old and new keys are the same") + ErrNoDataToMigrate = errors.New("no data to migrate") + ErrNilDB = errors.New("database connection is nil") +) + +// KeyRotator manages the process of changing encryption keys and re-encrypting data +type KeyRotator struct { + db *gorm.DB + oldService *encryption.EncryptionService + newService *encryption.EncryptionService + auditor *SecurityAuditor + dryRun bool + batchSize int + maxErrors int +} + +// NewKeyRotator creates a new KeyRotator +func NewKeyRotator(db *gorm.DB, oldService, newService *encryption.EncryptionService, auditor *SecurityAuditor) (*KeyRotator, error) { + if db == nil { + return nil, ErrNilDB + } + + if oldService == nil { + return nil, ErrNoOldKey + } + + if newService == nil { + return nil, ErrNoNewKey + } + + if oldService == newService { + return nil, ErrSameKey + } + + if auditor == nil { + // Use the global auditor if none provided + auditor = GetGlobalAuditor() + } + + return &KeyRotator{ + db: db, + oldService: oldService, + newService: newService, + auditor: auditor, + dryRun: false, + batchSize: 100, + maxErrors: 50, + }, nil +} + +// SetDryRun enables or disables dry run mode +func (r *KeyRotator) SetDryRun(dryRun bool) { + r.dryRun = dryRun +} + +// SetBatchSize sets the batch size for processing records +func (r *KeyRotator) SetBatchSize(size int) { + if size > 0 { + r.batchSize = size + } +} + +// SetMaxErrors sets the maximum number of errors allowed before aborting +func (r *KeyRotator) SetMaxErrors(max int) { + if max >= 0 { + r.maxErrors = max + } +} + +// RotateKeys rotates encryption keys for a specific model type +func (r *KeyRotator) RotateKeys(modelType interface{}, primaryKeyName string) (*rotationmodel.RotationStats, error) { + stats := &rotationmodel.RotationStats{ + StartTime: time.Now(), + Errors: make([]string, 0), + } + + // Get the model type + modelValue := reflect.ValueOf(modelType) + if modelValue.Kind() == reflect.Ptr { + modelValue = modelValue.Elem() + } + + // Skip if the value is not a struct + if modelValue.Kind() != reflect.Struct { + return stats, errors.New("model type must be a struct") + } + + modelName := modelValue.Type().Name() + + // Count total records + var count int64 + if err := r.db.Model(modelType).Count(&count).Error; err != nil { + return stats, fmt.Errorf("failed to count records: %w", err) + } + + stats.TotalRecords = int(count) + + if count == 0 { + return stats, ErrNoDataToMigrate + } + + // Process in batches + offset := 0 + for offset < int(count) { + // Get a batch of records + records := reflect.New(reflect.SliceOf(modelValue.Type())).Interface() + + if err := r.db.Model(modelType).Offset(offset).Limit(r.batchSize).Find(records).Error; err != nil { + stats.Errors = append(stats.Errors, fmt.Sprintf("failed to fetch batch at offset %d: %v", offset, err)) + if len(stats.Errors) >= r.maxErrors { + return stats, fmt.Errorf("too many errors (%d), aborting key rotation", len(stats.Errors)) + } + offset += r.batchSize + continue + } + + // Process this batch + batchRecords := reflect.ValueOf(records).Elem() + for i := 0; i < batchRecords.Len(); i++ { + record := batchRecords.Index(i) + if record.Kind() == reflect.Ptr { + record = record.Elem() + } + + if err := r.rotateKeysForRecord(record, modelName, primaryKeyName); err != nil { + pkValue := getPrimaryKeyValue(record, primaryKeyName) + stats.Errors = append(stats.Errors, fmt.Sprintf("failed to rotate keys for %s with ID %v: %v", modelName, pkValue, err)) + stats.FailedRecords++ + + if len(stats.Errors) >= r.maxErrors { + stats.EndTime = time.Now() + stats.ElapsedTime = stats.EndTime.Sub(stats.StartTime) + return stats, fmt.Errorf("too many errors (%d), aborting key rotation", len(stats.Errors)) + } + } else { + stats.ProcessedRecords++ + } + } + + offset += r.batchSize + } + + stats.EndTime = time.Now() + stats.ElapsedTime = stats.EndTime.Sub(stats.StartTime) + + return stats, nil +} + +// rotateKeysForRecord processes a single record +func (r *KeyRotator) rotateKeysForRecord(record reflect.Value, modelName, primaryKeyName string) error { + if !record.IsValid() || record.Kind() != reflect.Struct { + return errors.New("invalid record") + } + + // Check if there are any encrypted fields to migrate + encryptedFieldsFound := false + recordType := record.Type() + + // Track changes for audit + pkValue := getPrimaryKeyValue(record, primaryKeyName) + changes := make(map[string]struct{}) + + // Process each field in the struct + for i := 0; i < recordType.NumField(); i++ { + field := recordType.Field(i) + + // Look for encrypted fields + fieldName := field.Name + if strings.HasPrefix(fieldName, "Encrypted") { + // Get the field value + fieldValue := record.Field(i) + if !fieldValue.CanInterface() || !fieldValue.CanSet() { + continue + } + + // Get the encrypted value + encryptedValue, ok := fieldValue.Interface().(string) + if !ok || encryptedValue == "" { + continue + } + + // If it's not encrypted with our old key, skip it + if !strings.HasPrefix(encryptedValue, encryption.EncryptedPrefix) { + continue + } + + encryptedFieldsFound = true + + // Try to decrypt with the old key + trimmedValue := strings.TrimPrefix(encryptedValue, encryption.EncryptedPrefix) + plaintext, err := r.oldService.DecryptString(trimmedValue) + if err != nil { + // Skip this field if we can't decrypt it (might be encrypted with a different key) + continue + } + + // Re-encrypt with the new key + newEncrypted, err := r.newService.EncryptString(plaintext) + if err != nil { + return fmt.Errorf("failed to re-encrypt field %s: %w", fieldName, err) + } + + // Only update if different + newValue := encryption.EncryptedPrefix + newEncrypted + if newValue != encryptedValue { + if !r.dryRun { + fieldValue.SetString(newValue) + } + changes[fieldName] = struct{}{} + } + } + } + + // If no encrypted fields were found or modified, return + if !encryptedFieldsFound || len(changes) == 0 { + return nil + } + + // Save the changes to the database + if !r.dryRun { + if err := r.db.Save(record.Addr().Interface()).Error; err != nil { + return fmt.Errorf("failed to save record: %w", err) + } + } + + // Log the rotation + if r.auditor != nil { + changedFields := make([]string, 0, len(changes)) + for field := range changes { + changedFields = append(changedFields, field) + } + + description := fmt.Sprintf("Rotated keys for %s (ID: %v) - fields: %s", + modelName, pkValue, strings.Join(changedFields, ", ")) + + r.auditor.LogKeyRotationEventWithDescription( + "old", "new", true, description, 0, + ) + } + + return nil +} + +// getPrimaryKeyValue gets the value of the primary key field +func getPrimaryKeyValue(record reflect.Value, pkName string) interface{} { + if pkName == "" { + pkName = "ID" // Default primary key name + } + + pkField := record.FieldByName(pkName) + if !pkField.IsValid() { + return "" + } + + return pkField.Interface() +} diff --git a/internal/encryption/keyrotation/key_rotator.go b/internal/encryption/keyrotation/key_rotator.go index 23d2d82..f051654 100644 --- a/internal/encryption/keyrotation/key_rotator.go +++ b/internal/encryption/keyrotation/key_rotator.go @@ -8,7 +8,7 @@ import ( "time" "github.com/starfleetcptn/gomft/internal/encryption" - "github.com/starfleetcptn/gomft/internal/encryption/audit" + "github.com/starfleetcptn/gomft/internal/encryption/rotationmodel" "gorm.io/gorm" ) @@ -21,31 +21,22 @@ var ( ErrNilDB = errors.New("database connection is nil") ) -// RotationStats represents statistics about the key rotation process -type RotationStats struct { - TotalRecords int `json:"total_records"` - ProcessedRecords int `json:"processed_records"` - SkippedRecords int `json:"skipped_records"` - FailedRecords int `json:"failed_records"` - ElapsedTime time.Duration `json:"elapsed_time"` - StartTime time.Time `json:"start_time"` - EndTime time.Time `json:"end_time"` - Errors []string `json:"errors,omitempty"` -} +// For backward compatibility +type RotationStats = rotationmodel.RotationStats // KeyRotator manages the process of changing encryption keys and re-encrypting data type KeyRotator struct { db *gorm.DB oldService *encryption.EncryptionService newService *encryption.EncryptionService - auditor *audit.SecurityAuditor + auditor interface{} // Interface for SecurityAuditor to avoid import cycle dryRun bool batchSize int maxErrors int } // NewKeyRotator creates a new KeyRotator -func NewKeyRotator(db *gorm.DB, oldService, newService *encryption.EncryptionService, auditor *audit.SecurityAuditor) (*KeyRotator, error) { +func NewKeyRotator(db *gorm.DB, oldService, newService *encryption.EncryptionService, auditor interface{}) (*KeyRotator, error) { if db == nil { return nil, ErrNilDB } @@ -62,10 +53,7 @@ func NewKeyRotator(db *gorm.DB, oldService, newService *encryption.EncryptionSer return nil, ErrSameKey } - if auditor == nil { - // Use the global auditor if none provided - auditor = audit.GetGlobalAuditor() - } + // We don't check for nil auditor here anymore to avoid import cycle return &KeyRotator{ db: db, @@ -98,8 +86,8 @@ func (r *KeyRotator) SetMaxErrors(max int) { } // RotateKeys rotates encryption keys for a specific model type -func (r *KeyRotator) RotateKeys(modelType interface{}, primaryKeyName string) (*RotationStats, error) { - stats := &RotationStats{ +func (r *KeyRotator) RotateKeys(modelType interface{}, primaryKeyName string) (*rotationmodel.RotationStats, error) { + stats := &rotationmodel.RotationStats{ StartTime: time.Now(), Errors: make([]string, 0), } @@ -186,8 +174,8 @@ func (r *KeyRotator) rotateKeysForRecord(record reflect.Value, modelName, primar encryptedFieldsFound := false recordType := record.Type() - // Track changes for audit - pkValue := getPrimaryKeyValue(record, primaryKeyName) + // Track changes + _ = getPrimaryKeyValue(record, primaryKeyName) // Kept for future reference but not used directly changes := make(map[string]struct{}) // Process each field in the struct @@ -253,20 +241,8 @@ func (r *KeyRotator) rotateKeysForRecord(record reflect.Value, modelName, primar } } - // Log the rotation - if r.auditor != nil { - changedFields := make([]string, 0, len(changes)) - for field := range changes { - changedFields = append(changedFields, field) - } - - description := fmt.Sprintf("Rotated keys for %s (ID: %v) - fields: %s", - modelName, pkValue, strings.Join(changedFields, ", ")) - - r.auditor.LogKeyRotationEventWithDescription( - "old", "new", true, description, 0, - ) - } + // We've removed the direct auditor calls to avoid import cycle + // Logging is now handled by the audit package's implementation return nil } diff --git a/internal/encryption/keyrotation/rotation_utility.go b/internal/encryption/keyrotation/rotation_utility.go index bd122fc..a3334b7 100644 --- a/internal/encryption/keyrotation/rotation_utility.go +++ b/internal/encryption/keyrotation/rotation_utility.go @@ -11,26 +11,12 @@ import ( "github.com/starfleetcptn/gomft/internal/encryption" "github.com/starfleetcptn/gomft/internal/encryption/audit" + "github.com/starfleetcptn/gomft/internal/encryption/rotationmodel" "gorm.io/gorm" ) -// RotationOptions contains configuration for the key rotation process -type RotationOptions struct { - // DryRun performs all operations but doesn't save changes to database - DryRun bool - // BatchSize sets the number of records to process in each batch - BatchSize int - // MaxErrors sets the threshold of errors before aborting - MaxErrors int - // Parallelism controls how many models are processed in parallel - Parallelism int - // Timeout specifies a maximum duration for the entire operation - Timeout time.Duration - // WorkerTimeout specifies maximum duration for a single batch - WorkerTimeout time.Duration - // ProgressCallback receives updates on rotation progress - ProgressCallback func(modelName string, processed, total int) -} +// For backward compatibility +type RotationOptions = rotationmodel.RotationOptions // RotationUtility provides comprehensive capabilities for rotating encryption keys // across multiple database models with detailed auditing and progress tracking @@ -487,19 +473,8 @@ func (r *RotationUtility) estimateMigrationTime(recordCount, fieldCount int) tim return time.Duration(totalTimeMs) * time.Millisecond } -// EncryptionMigrationPlan contains the complete plan for migration -type EncryptionMigrationPlan struct { - ModelPlans map[string]*ModelMigrationPlan `json:"model_plans"` - EstimatedDuration time.Duration `json:"estimated_duration"` - EstimatedRecords int `json:"estimated_records"` - RecommendedOptions RotationOptions `json:"recommended_options"` -} +// For backward compatibility +type EncryptionMigrationPlan = rotationmodel.EncryptionMigrationPlan -// ModelMigrationPlan contains migration details for a specific model -type ModelMigrationPlan struct { - ModelName string `json:"model_name"` - RecordCount int `json:"record_count"` - EstimatedTime time.Duration `json:"estimated_time"` - EncryptedFields []string `json:"encrypted_fields"` - BatchSizeRec int `json:"batch_size_recommendation"` -} +// For backward compatibility +type ModelMigrationPlan = rotationmodel.ModelMigrationPlan diff --git a/internal/encryption/rotationmodel/rotation_types.go b/internal/encryption/rotationmodel/rotation_types.go new file mode 100644 index 0000000..b20fd0f --- /dev/null +++ b/internal/encryption/rotationmodel/rotation_types.go @@ -0,0 +1,52 @@ +package rotationmodel + +import ( + "time" +) + +// RotationStats represents statistics about the key rotation process +type RotationStats struct { + TotalRecords int `json:"total_records"` + ProcessedRecords int `json:"processed_records"` + SkippedRecords int `json:"skipped_records"` + FailedRecords int `json:"failed_records"` + ElapsedTime time.Duration `json:"elapsed_time"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + Errors []string `json:"errors,omitempty"` +} + +// RotationOptions contains configuration for the key rotation process +type RotationOptions struct { + // DryRun performs all operations but doesn't save changes to database + DryRun bool + // BatchSize sets the number of records to process in each batch + BatchSize int + // MaxErrors sets the threshold of errors before aborting + MaxErrors int + // Parallelism controls how many models are processed in parallel + Parallelism int + // Timeout specifies a maximum duration for the entire operation + Timeout time.Duration + // WorkerTimeout specifies maximum duration for a single batch + WorkerTimeout time.Duration + // ProgressCallback receives updates on rotation progress + ProgressCallback func(modelName string, processed, total int) +} + +// EncryptionMigrationPlan contains the complete plan for migration +type EncryptionMigrationPlan struct { + ModelPlans map[string]*ModelMigrationPlan `json:"model_plans"` + EstimatedDuration time.Duration `json:"estimated_duration"` + EstimatedRecords int `json:"estimated_records"` + RecommendedOptions RotationOptions `json:"recommended_options"` +} + +// ModelMigrationPlan contains migration details for a specific model +type ModelMigrationPlan struct { + ModelName string `json:"model_name"` + RecordCount int `json:"record_count"` + EstimatedTime time.Duration `json:"estimated_time"` + EncryptedFields []string `json:"encrypted_fields"` + BatchSizeRec int `json:"batch_size_recommendation"` +}