From 1b1d043d50873b99de024dced7ae89c9a1eef99f Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sat, 8 Mar 2025 15:11:47 -0800 Subject: [PATCH] feat: Add support for WebDAV, NextCloud, OneDrive, and Google Drive storage - Update config_form.templ to include form fields for new storage types - Modify db.go to support generating rclone configs for WebDAV, NextCloud, OneDrive, and Google Drive - Add migration to include new cloud storage fields in transfer_configs table - Update migrations.go to include new cloud storage migration --- components/config_form.templ | 604 +++++++++++++++++- internal/db/db.go | 144 +++++ .../db/migrations/add_cloud_storage_fields.go | 63 ++ internal/db/migrations/migrations.go | 3 +- 4 files changed, 808 insertions(+), 6 deletions(-) create mode 100644 internal/db/migrations/add_cloud_storage_fields.go diff --git a/components/config_form.templ b/components/config_form.templ index 0e9341e..4d49ac4 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -38,6 +38,11 @@ func getInitialData(config *db.TransferConfig) string { sourceDomain := "" // FTP fields sourcePassiveMode := true + // OneDrive and Google Drive fields + sourceClientId := "" + sourceClientSecret := "" + sourceDriveId := "" + sourceTeamDrive := "" filePattern := "*" outputPattern := "" destinationType := "local" @@ -58,6 +63,11 @@ func getInitialData(config *db.TransferConfig) string { destDomain := "" // FTP fields destPassiveMode := true + // OneDrive and Google Drive fields + destClientId := "" + destClientSecret := "" + destDriveId := "" + destTeamDrive := "" archivePath := "" archiveEnabled := false deleteAfterTransfer := false @@ -83,6 +93,11 @@ func getInitialData(config *db.TransferConfig) string { sourceDomain = config.SourceDomain // FTP fields sourcePassiveMode = config.SourcePassiveMode + // OneDrive and Google Drive fields + sourceClientId = config.SourceClientID + sourceClientSecret = config.SourceClientSecret + sourceDriveId = config.SourceDriveID + sourceTeamDrive = config.SourceTeamDrive filePattern = config.FilePattern outputPattern = config.OutputPattern destinationType = config.DestinationType @@ -103,6 +118,11 @@ func getInitialData(config *db.TransferConfig) string { destDomain = config.DestDomain // FTP fields destPassiveMode = config.DestPassiveMode + // OneDrive and Google Drive fields + destClientId = config.DestClientID + destClientSecret = config.DestClientSecret + destDriveId = config.DestDriveID + destTeamDrive = config.DestTeamDrive archivePath = config.ArchivePath archiveEnabled = config.ArchiveEnabled deleteAfterTransfer = config.DeleteAfterTransfer @@ -126,6 +146,10 @@ func getInitialData(config *db.TransferConfig) string { sourceShare: '%s', sourceDomain: '%s', sourcePassiveMode: %v, + sourceClientId: '%s', + sourceClientSecret: '%s', + sourceDriveId: '%s', + sourceTeamDrive: '%s', filePattern: '%s', outputPattern: '%s', destinationType: '%s', @@ -146,12 +170,18 @@ func getInitialData(config *db.TransferConfig) string { destDomain: '%s', // FTP fields destPassiveMode: %v, + // OneDrive and Google Drive fields + destClientId: '%s', + destClientSecret: '%s', + destDriveId: '%s', + destTeamDrive: '%s', // Existing fields archivePath: '%s', archiveEnabled: %v, deleteAfterTransfer: %v, rcloneFlags: '%s', loading: false, + validate() { if (this.sourceType === 'sftp') { if (!this.sourceHost || !this.sourceUser || (!this.sourcePassword && !this.sourceKeyFile)) { @@ -177,6 +207,22 @@ func getInitialData(config *db.TransferConfig) string { if (!this.sourceHost || !this.sourceUser || !this.sourcePassword) { return false; } + } else if (this.sourceType === 'webdav') { + if (!this.sourceHost || !this.sourceUser || !this.sourcePassword) { + return false; + } + } else if (this.sourceType === 'nextcloud') { + if (!this.sourceHost || !this.sourceUser || !this.sourcePassword) { + return false; + } + } else if (this.sourceType === 'onedrive') { + if (!this.sourceClientId || !this.sourceClientSecret) { + return false; + } + } else if (this.sourceType === 'google_drive') { + if (!this.sourceClientId || !this.sourceClientSecret) { + return false; + } } if (this.destinationType === 'sftp') { @@ -203,15 +249,33 @@ func getInitialData(config *db.TransferConfig) string { if (!this.destHost || !this.destUser || !this.destPassword) { return false; } + } else if (this.destinationType === 'webdav') { + if (!this.destHost || !this.destUser || !this.destPassword) { + return false; + } + } else if (this.destinationType === 'nextcloud') { + if (!this.destHost || !this.destUser || !this.destPassword) { + return false; + } + } else if (this.destinationType === 'onedrive') { + if (!this.destClientId || !this.destClientSecret) { + return false; + } + } else if (this.destinationType === 'google_drive') { + if (!this.destClientId || !this.destClientSecret) { + return false; + } } return this.name && this.sourcePath && this.filePattern && this.destinationPath && (!this.archiveEnabled || this.archivePath); } }`, name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceBucket, sourceRegion, sourceAccessKey, sourceSecretKey, sourceEndpoint, sourceShare, sourceDomain, sourcePassiveMode, + sourceClientId, sourceClientSecret, sourceDriveId, sourceTeamDrive, filePattern, outputPattern, destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode, + destClientId, destClientSecret, destDriveId, destTeamDrive, archivePath, archiveEnabled, deleteAfterTransfer, rcloneFlags) } @@ -284,6 +348,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + + + @@ -728,6 +796,268 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + + + + + + +
@@ -814,6 +1144,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + + +
@@ -1258,15 +1592,277 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + + + + + + +
-
-

Location to move files after successful transfer

@@ -1339,7 +1934,6 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500" placeholder="--transfers 4 --checkers 8"/> -

Optional: Additional rclone flags for fine-tuning the transfer.

diff --git a/internal/db/db.go b/internal/db/db.go index c02adb9..1779f94 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -66,6 +66,11 @@ type TransferConfig struct { SourceDomain string `form:"source_domain"` // FTP source fields SourcePassiveMode bool `gorm:"default:true" form:"source_passive_mode"` + // OneDrive and Google Drive source fields + SourceClientID string `form:"source_client_id"` + SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form + SourceDriveID string `form:"source_drive_id"` // For OneDrive + SourceTeamDrive string `form:"source_team_drive"` // For Google Drive // General fields FilePattern string `gorm:"default:'*'" form:"file_pattern"` OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables @@ -87,6 +92,11 @@ type TransferConfig struct { DestDomain string `form:"dest_domain"` // FTP destination fields DestPassiveMode bool `gorm:"default:true" form:"dest_passive_mode"` + // OneDrive and Google Drive destination fields + DestClientID string `form:"dest_client_id"` + DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form + DestDriveID string `form:"dest_drive_id"` // For OneDrive + DestTeamDrive string `form:"dest_team_drive"` // For Google Drive // General fields ArchivePath string `form:"archive_path"` ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"` @@ -426,6 +436,73 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "passive", "true") } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "webdav": + args := []string{ + "config", "create", sourceName, "webdav", + "url", config.SourceHost, + "user", config.SourceUser, + "pass", config.SourcePassword, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "nextcloud": + args := []string{ + "config", "create", sourceName, "webdav", + "url", config.SourceHost, + "user", config.SourceUser, + "pass", config.SourcePassword, + "vendor", "nextcloud", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "onedrive": + args := []string{ + "config", "create", sourceName, "onedrive", + "client_id", config.SourceClientID, + "client_secret", config.SourceClientSecret, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + if config.SourceDriveID != "" { + args = append(args, "drive_id", config.SourceDriveID) + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "google_drive": + args := []string{ + "config", "create", sourceName, "drive", + "client_id", config.SourceClientID, + "client_secret", config.SourceClientSecret, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + if config.SourceTeamDrive != "" { + args = append(args, "team_drive", config.SourceTeamDrive) + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) @@ -547,6 +624,73 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "passive", "true") } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "webdav": + args := []string{ + "config", "create", destName, "webdav", + "url", config.DestHost, + "user", config.DestUser, + "pass", config.DestPassword, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "nextcloud": + args := []string{ + "config", "create", destName, "webdav", + "url", config.DestHost, + "user", config.DestUser, + "pass", config.DestPassword, + "vendor", "nextcloud", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "onedrive": + args := []string{ + "config", "create", destName, "onedrive", + "client_id", config.DestClientID, + "client_secret", config.DestClientSecret, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + if config.DestDriveID != "" { + args = append(args, "drive_id", config.DestDriveID) + } + + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "google_drive": + args := []string{ + "config", "create", destName, "drive", + "client_id", config.DestClientID, + "client_secret", config.DestClientSecret, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + if config.DestTeamDrive != "" { + args = append(args, "team_drive", config.DestTeamDrive) + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) diff --git a/internal/db/migrations/add_cloud_storage_fields.go b/internal/db/migrations/add_cloud_storage_fields.go new file mode 100644 index 0000000..3582df0 --- /dev/null +++ b/internal/db/migrations/add_cloud_storage_fields.go @@ -0,0 +1,63 @@ +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddCloudStorageFields adds fields for WebDAV, NextCloud, OneDrive, and Google Drive +func AddCloudStorageFields() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "add_cloud_storage_fields", + Migrate: func(tx *gorm.DB) error { + // Add source fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_client_id VARCHAR(255)").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_drive_id VARCHAR(255)").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_team_drive VARCHAR(255)").Error; err != nil { + return err + } + + // Add destination fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_client_id VARCHAR(255)").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_drive_id VARCHAR(255)").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_team_drive VARCHAR(255)").Error; err != nil { + return err + } + + return nil + }, + Rollback: func(tx *gorm.DB) error { + // Drop source fields + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_client_id").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_drive_id").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_team_drive").Error; err != nil { + return err + } + + // Drop destination fields + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_client_id").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_drive_id").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_team_drive").Error; err != nil { + return err + } + + return nil + }, + } +} \ No newline at end of file diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 31161b3..c7705a7 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -6,10 +6,11 @@ import ( ) // InitMigrations initializes the migrations -func InitMigrations() *gormigrate.Gormigrate { +func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate { migrations := []*gormigrate.Migration{ // ... existing migrations AddDeleteAfterTransferColumn(), + AddCloudStorageFields(), } return gormigrate.New(db, gormigrate.DefaultOptions, migrations)