From 8607f2098ad7c0fbf5ef93be7777f4bbfea6a43b Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sat, 15 Mar 2025 17:36:36 -0700 Subject: [PATCH 1/3] feat: Integrate Google Drive support and enhance configuration handling - Add Google Drive as a source and destination option in the configuration forms. - Implement Google Drive authentication flow and token management. - Update job and configuration handlers to support Google Drive-specific settings. - Enhance UI components to include Google Drive configuration templates. - Introduce new tests for Google Drive integration and ensure proper handling of authentication and configuration. - Update database migrations to accommodate new fields related to Google Drive configurations. --- components/admin_tools.templ | 3 +- components/config_form.templ | 21 +- components/configs.templ | 56 ++ components/job_form.templ | 53 +- components/jobs.templ | 2 +- components/profile.templ | 2 +- components/providers/common/common.templ | 2 + components/providers/destination/gdrive.templ | 178 +++++ components/providers/providers_test.go | 10 + components/providers/source/gdrive.templ | 129 ++++ components/users.templ | 2 +- go.mod | 4 + go.sum | 8 + internal/api/api.go | 34 +- internal/db/db.go | 628 +++++++++++++++++- internal/db/db_test.go | 609 ++++++++++++++++- internal/db/edge_cases_test.go | 6 +- internal/db/error_handling_test.go | 4 +- internal/db/initialization_test.go | 2 +- .../add_google_drive_authenticated.go | 21 + internal/db/migrations/migrations.go | 1 + internal/db/rclone_test.go | 114 ++++ internal/db/transaction_test.go | 4 +- internal/scheduler/mock_scheduler.go | 2 +- internal/scheduler/mock_scheduler_test.go | 4 +- internal/scheduler/scheduler.go | 14 +- internal/scheduler/scheduler_test.go | 57 +- .../scheduler/webhook_integration_test.go | 30 +- internal/scheduler/webhook_test.go | 54 +- internal/testutils/testutils.go | 2 +- internal/web/handlers/admin_tools_handlers.go | 43 +- .../web/handlers/admin_tools_handlers_test.go | 58 +- internal/web/handlers/api_handlers.go | 2 +- internal/web/handlers/api_handlers_test.go | 23 +- internal/web/handlers/auth_handlers.go | 89 +-- internal/web/handlers/auth_handlers_test.go | 26 +- internal/web/handlers/config_handlers.go | 52 +- internal/web/handlers/config_handlers_test.go | 2 +- .../web/handlers/dashboard_handlers_test.go | 2 +- .../web/handlers/file_metadata_handlers.go | 11 +- .../handlers/file_metadata_handlers_test.go | 2 +- internal/web/handlers/gdrive_handlers.go | 380 +++++++++++ internal/web/handlers/import_jobs_test.go | 8 +- internal/web/handlers/job_handlers.go | 34 + internal/web/handlers/job_handlers_test.go | 68 +- internal/web/handlers/oauth_handlers.go | 239 +++++++ internal/web/handlers/routes.go | 6 + internal/web/handlers/test_utils.go | 2 +- internal/web/handlers/user_handlers.go | 11 +- internal/web/handlers/user_handlers_test.go | 2 +- internal/web/handlers/webhook_test.go | 30 +- main.go | 4 +- 52 files changed, 2795 insertions(+), 355 deletions(-) create mode 100644 components/providers/destination/gdrive.templ create mode 100644 components/providers/source/gdrive.templ create mode 100644 internal/db/migrations/add_google_drive_authenticated.go create mode 100644 internal/web/handlers/gdrive_handlers.go create mode 100644 internal/web/handlers/oauth_handlers.go diff --git a/components/admin_tools.templ b/components/admin_tools.templ index 7a51949..dae3fee 100644 --- a/components/admin_tools.templ +++ b/components/admin_tools.templ @@ -85,7 +85,8 @@ script hideDialog(id string) { } script submitFormAndHideDialog(formId string, dialogId string) { - document.getElementById(formId).submit(); + // Use HTMX's API to trigger the request instead of bypassing it + htmx.trigger(document.getElementById(formId), 'submit'); document.getElementById(dialogId).classList.add("hidden"); } diff --git a/components/config_form.templ b/components/config_form.templ index ae4f575..d392ba8 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -75,6 +75,7 @@ func getInitialData(config *db.TransferConfig) string { skipProcessedFiles := true maxConcurrentTransfers := 4 rcloneFlags := "" + useBuiltinAuth := true // If editing an existing config, populate with those values if config != nil { @@ -96,7 +97,7 @@ func getInitialData(config *db.TransferConfig) string { sourceEndpoint = config.SourceEndpoint sourceShare = config.SourceShare sourceDomain = config.SourceDomain - sourcePassiveMode = config.SourcePassiveMode + sourcePassiveMode = config.GetSourcePassiveMode() sourceClientId = config.SourceClientID sourceClientSecret = config.SourceClientSecret sourceDriveId = config.SourceDriveID @@ -122,18 +123,21 @@ func getInitialData(config *db.TransferConfig) string { destEndpoint = config.DestEndpoint destShare = config.DestShare destDomain = config.DestDomain - destPassiveMode = config.DestPassiveMode + destPassiveMode = config.GetDestPassiveMode() destClientId = config.DestClientID destClientSecret = config.DestClientSecret destDriveId = config.DestDriveID destTeamDrive = config.DestTeamDrive archivePath = config.ArchivePath - archiveEnabled = config.ArchiveEnabled - deleteAfterTransfer = config.DeleteAfterTransfer + archiveEnabled = config.GetArchiveEnabled() + deleteAfterTransfer = config.GetDeleteAfterTransfer() skipProcessedFiles = config.GetSkipProcessedFiles() maxConcurrentTransfers = config.MaxConcurrentTransfers rcloneFlags = config.RcloneFlags + if destClientId != "" || destClientSecret != "" { + useBuiltinAuth = false + } } // Return the JSON-formatted string with all the data @@ -183,6 +187,7 @@ func getInitialData(config *db.TransferConfig) string { destClientSecret: '%s', destDriveId: '%s', destTeamDrive: '%s', + useBuiltinAuth: %v, archivePath: '%s', archiveEnabled: %v, @@ -198,6 +203,7 @@ func getInitialData(config *db.TransferConfig) string { destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType, destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode, destClientId, destClientSecret, destDriveId, destTeamDrive, + useBuiltinAuth, archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags) } @@ -277,6 +283,9 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @source.NextCloudSourceForm() + @common.FilePatternFields() @@ -316,6 +325,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + diff --git a/components/configs.templ b/components/configs.templ index f685849..a15ded9 100644 --- a/components/configs.templ +++ b/components/configs.templ @@ -68,6 +68,9 @@ script triggerConfigDelete(dialogId string, configID uint, configName string) { type ConfigsData struct { Configs []db.TransferConfig + Error string + ErrorDetails string + Status string } templ Configs(ctx context.Context, data ConfigsData) { @@ -106,6 +109,27 @@ templ Configs(ctx context.Context, data ConfigsData) { console.log("Notyf initialized:", window.notyf); } + // Show status messages based on URL parameters + document.addEventListener('DOMContentLoaded', function() { + // Check for error message + const urlParams = new URLSearchParams(window.location.search); + const errorMsg = urlParams.get('error'); + const errorDetails = urlParams.get('details'); + const status = urlParams.get('status'); + + if (errorMsg) { + let message = errorMsg; + if (errorDetails) { + message += ": " + errorDetails; + } + window.notyf.error(message); + } + + if (status === 'gdrive_auth_success') { + window.notyf.success("Google Drive authentication completed successfully"); + } + }); + // Track all HTMX events for debugging document.addEventListener('htmx:beforeRequest', function(event) { @@ -244,8 +268,32 @@ templ Configs(ctx context.Context, data ConfigsData) {

{ config.Name }

+ + + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && !config.GetGoogleDriveAuthenticated() { + + + Authentication Required + + } + + + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && config.GetGoogleDriveAuthenticated() { + + + Authenticated + + }
+ + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive") && !config.GetGoogleDriveAuthenticated() { + + + Authenticate + + } + Edit @@ -304,6 +352,14 @@ templ Configs(ctx context.Context, data ConfigsData) { Configurations define how files are transferred between systems

+ + +
+

+ + Google Drive configurations require authentication. Click the "Authenticate" button to complete setup. +

+
} diff --git a/components/job_form.templ b/components/job_form.templ index 92ff20e..9e09ee3 100644 --- a/components/job_form.templ +++ b/components/job_form.templ @@ -191,16 +191,17 @@ templ JobForm(ctx context.Context, data JobFormData) {
+ if data.Job.GetEnabled() { + checked + } + class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded" />
-

- - Disabled jobs will not run automatically. +

+ Jobs that are not enabled will not run automatically on schedule.

@@ -215,12 +216,15 @@ templ JobForm(ctx context.Context, data JobFormData) {
+ if data.Job.GetWebhookEnabled() { + checked + } + class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded" />
@@ -293,7 +297,9 @@ templ JobForm(ctx context.Context, data JobFormData) { id="notify_on_success" name="notify_on_success" value="true" - checked + if data.Job.GetNotifyOnSuccess() { + checked + } class="h-4 w-4 text-primary-600 focus:ring-primary-500 border-secondary-300 dark:border-secondary-700 rounded"/>
- if (config.DestinationType == "gdrive" || config.SourceType == "gdrive") && !config.GetGoogleDriveAuthenticated() { + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && !config.GetGoogleAuthenticated() { Authenticate diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index 0ffddb5..8562ee2 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -195,6 +195,7 @@ templ SourceSelection() { +
@@ -219,6 +220,7 @@ templ DestinationSelection() { + diff --git a/components/providers/destination/gphotos.templ b/components/providers/destination/gphotos.templ new file mode 100644 index 0000000..7fc56f7 --- /dev/null +++ b/components/providers/destination/gphotos.templ @@ -0,0 +1,220 @@ +package destination + +templ GooglePhotosDestinationForm() { +
+
+ +

+ Simple one-click authentication using rclone's shared credentials +

+
+ +
+
+ +
+
+ +
+ +
+

+ Client ID from Google Cloud Console +

+
+ +
+ +
+
+ +
+ +
+

+ Client Secret from Google Cloud Console +

+
+
+ +
+ +

+ Only request read-only access to your photos +

+
+ +
+ +
+
+ +
+ +
+

+ Only include photos uploaded after this year +

+
+ +
+ +

+ Include archived photos and videos in media listings +

+
+ +
+ +
+
+ +
+ +
+

+ Path within Google Photos where files will be uploaded +

+
+ +
+
+
+ +
+
+

Important: Authentication Required

+
+

After saving this configuration, you will need to authenticate with Google Photos.

+

The authentication process will require you to:

+
    +
  1. Visit a Google authorization URL
    2. +
  2. Sign in to your Google account
    3. +
  3. Grant permission to access your Google Photos
    4. +
  4. Copy the authorization code back to this application
    5. +
+

+ This is a one-time process for each configuration. The application will store your authorization token securely. +

+
+
+
+
+ +
+
+
+ +
+
+

Authentication Information

+
+ + +
+
+
+
+ +
+
+
+ +
+
+

Important Note About Google Photos

+
+

All media items uploaded to Google Photos with rclone are stored in full resolution at original quality. These uploads will count towards storage in your Google Account.

+
+
+
+
+
+} \ No newline at end of file diff --git a/components/providers/source/gphotos.templ b/components/providers/source/gphotos.templ new file mode 100644 index 0000000..d1e0187 --- /dev/null +++ b/components/providers/source/gphotos.templ @@ -0,0 +1,220 @@ +package source + +templ GooglePhotosSourceForm() { +
+
+ +

+ Simple one-click authentication using rclone's shared credentials +

+
+ +
+
+ +
+
+ +
+ +
+

+ Client ID from Google Cloud Console +

+
+ +
+ +
+
+ +
+ +
+

+ Client Secret from Google Cloud Console +

+
+
+ +
+ +

+ Only request read-only access to your photos +

+
+ +
+ +
+
+ +
+ +
+

+ Only include photos uploaded after this year +

+
+ +
+ +

+ Include archived photos and videos in media listings +

+
+ +
+ +
+
+ +
+ +
+

+ Path within Google Photos to download files from +

+
+ +
+
+
+ +
+
+

Important: Authentication Required

+
+

After saving this configuration, you will need to authenticate with Google Photos.

+

The authentication process will require you to:

+
    +
  1. Visit a Google authorization URL
    2. +
  2. Sign in to your Google account
    3. +
  3. Grant permission to access your Google Photos
    4. +
  4. Copy the authorization code back to this application
    5. +
+

+ This is a one-time process for each configuration. The application will store your authorization token securely. +

+
+
+
+
+ +
+
+
+ +
+
+

Authentication Information

+
+ + +
+
+
+
+ +
+
+
+ +
+
+

Important Note About Google Photos

+
+

When downloading from Google Photos, be aware that some original metadata may not be preserved. Google Photos processes and may compress some images upon upload.

+
+
+
+
+
+} \ No newline at end of file diff --git a/internal/db/db.go b/internal/db/db.go index 527b4a6..09e295b 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,11 +1,11 @@ package db import ( - "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -74,6 +74,10 @@ type TransferConfig struct { SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form SourceDriveID string `form:"source_drive_id"` // For OneDrive SourceTeamDrive string `form:"source_team_drive"` // For Google Drive + // Google Photos source fields + SourceReadOnly *bool `form:"source_read_only"` // For Google Photos + SourceStartYear int `form:"source_start_year"` // For Google Photos + SourceIncludeArchived *bool `form:"source_include_archived"` // For Google Photos // General fields FilePattern string `gorm:"default:'*'" form:"file_pattern"` OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables @@ -100,8 +104,13 @@ type TransferConfig struct { DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form DestDriveID string `form:"dest_drive_id"` // For OneDrive DestTeamDrive string `form:"dest_team_drive"` // For Google Drive - // Google Drive authentication status - GoogleDriveAuthenticated *bool `gorm:"default:false"` + // Google Photos destination fields + DestReadOnly *bool `form:"dest_read_only"` // For Google Photos + DestStartYear int `form:"dest_start_year"` // For Google Photos + DestIncludeArchived *bool `form:"dest_include_archived"` // For Google Photos + // Security fields + UseBuiltinAuth *bool `form:"use_builtin_auth"` // For Google and other OAuth services + GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed // General fields ArchivePath string `form:"archive_path"` ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"` @@ -632,6 +641,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "team_drive", config.SourceTeamDrive) } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "gphotos": + args := []string{ + "config", "create", sourceName, "google photos", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + // Only add client_id and client_secret if they're provided (not empty) + // This allows using rclone's built-in authentication + if config.SourceClientID != "" && config.SourceClientSecret != "" { + args = append(args, "client_id", config.SourceClientID) + args = append(args, "client_secret", config.SourceClientSecret) + } + + // Add read_only option if specified + if config.SourceReadOnly != nil && *config.SourceReadOnly { + args = append(args, "read_only", "true") + } + + // Add start_year if specified + if config.SourceStartYear > 0 { + args = append(args, "start_year", strconv.Itoa(config.SourceStartYear)) + } + + // Add include_archived if specified + if config.SourceIncludeArchived != nil && *config.SourceIncludeArchived { + args = append(args, "include_archived", "true") + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) @@ -829,6 +872,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "root_folder_id", config.DestDriveID) } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "gphotos": + args := []string{ + "config", "create", destName, "google photos", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + // Only add client_id and client_secret if they're provided (not empty) + // This allows using rclone's built-in authentication + if config.DestClientID != "" && config.DestClientSecret != "" { + args = append(args, "client_id", config.DestClientID) + args = append(args, "client_secret", config.DestClientSecret) + } + + // Add read_only option if specified + if config.DestReadOnly != nil && *config.DestReadOnly { + args = append(args, "read_only", "true") + } + + // Add start_year if specified + if config.DestStartYear > 0 { + args = append(args, "start_year", strconv.Itoa(config.DestStartYear)) + } + + // Add include_archived if specified + if config.DestIncludeArchived != nil && *config.DestIncludeArchived { + args = append(args, "include_archived", "true") + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) @@ -1010,235 +1087,129 @@ func (db *DB) StoreGoogleDriveToken(configIDStr string, token string) error { // GenerateRcloneConfigWithToken generates a rclone config file for a transfer config with a provided token func (db *DB) GenerateRcloneConfigWithToken(config *TransferConfig, token string) error { - // Ensure we have a config directory - dataDir := os.Getenv("DATA_DIR") - if dataDir == "" { - dataDir = "./data" - } - configDir := filepath.Join(dataDir, "configs") - if err := os.MkdirAll(configDir, 0755); err != nil { - return err + // Get the config path + configPath := db.GetConfigRclonePath(config) + if configPath == "" { + return fmt.Errorf("failed to get config path") } - // Generate rclone config based on the config type - configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", config.ID)) + // Clean up the token to ensure it's a single line JSON + token = strings.TrimSpace(token) + token = strings.ReplaceAll(token, "\n", "") + token = strings.ReplaceAll(token, "\r", "") - // Create a new config content - var configContent strings.Builder + // Determine if this is a source or destination config + var configType, section, clientID, clientSecret string + var readOnly, includeArchived *bool + var startYear int - // First add the source configuration - sourceName := fmt.Sprintf("source_%d", config.ID) + if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" { + configType = config.DestinationType + section = "dest" + clientID = config.DestClientID + clientSecret = config.DestClientSecret + readOnly = config.DestReadOnly + startYear = config.DestStartYear + includeArchived = config.DestIncludeArchived + } else if config.SourceType == "gdrive" || config.SourceType == "gphotos" { + configType = config.SourceType + section = "source" + clientID = config.SourceClientID + clientSecret = config.SourceClientSecret + readOnly = config.SourceReadOnly + startYear = config.SourceStartYear + includeArchived = config.SourceIncludeArchived + } else { + return fmt.Errorf("config is not for Google Drive or Google Photos") + } - // Handle the source configuration based on type - switch config.SourceType { - case "google_drive": - // Create a Google Drive remote using the "source_ID" naming convention for sources - sourceSection := fmt.Sprintf("[%s]\ntype = drive\n", sourceName) + // Read the existing config + content, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read config file: %v", err) + } - // Add custom client ID and secret if provided - if config.SourceClientID != "" && config.SourceClientSecret != "" { - sourceSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n", - config.SourceClientID, config.SourceClientSecret) + // Prepare the section content + var sectionContent string + if configType == "gdrive" { + sectionContent = fmt.Sprintf("[%s_%d]\ntype = drive\n", section, config.ID) + if clientID != "" { + sectionContent += fmt.Sprintf("client_id = %s\n", clientID) } + if clientSecret != "" { + sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret) + } + sectionContent += fmt.Sprintf("token = %s\n", token) // Add team drive if specified - if config.SourceTeamDrive != "" { - sourceSection += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive) + if section == "source" && config.SourceTeamDrive != "" { + sectionContent += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive) + } else if section == "dest" && config.DestTeamDrive != "" { + sectionContent += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive) } - // Clean up token string to prevent syntax errors and ensure it's a single line JSON - // First, remove any whitespace from the beginning and end - cleanToken := strings.TrimSpace(token) + // Add read-only flag if specified + if readOnly != nil && *readOnly { + sectionContent += "read_only = true\n" + } + } else if configType == "gphotos" { + sectionContent = fmt.Sprintf("[%s_%d]\ntype = google photos\n", section, config.ID) + if clientID != "" { + sectionContent += fmt.Sprintf("client_id = %s\n", clientID) + } + if clientSecret != "" { + sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret) + } + sectionContent += fmt.Sprintf("token = %s\n", token) - // Check if it's already a JSON object - if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") { - // It's a JSON object, but we need to make sure it's a single line - var jsonObj map[string]interface{} - if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil { - // Successfully parsed the JSON, now re-marshal it as a compact single line - compactJSON, err := json.Marshal(jsonObj) - if err == nil { - // Use the compact JSON as the token - sourceSection += fmt.Sprintf("token = %s\n", string(compactJSON)) - } else { - // If there was an error re-marshaling, use the original but remove newlines - // Replace all newlines and carriage returns with empty string - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // If we couldn't parse the JSON, just remove newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // Not a valid JSON, try to fix it - // First ensure it starts and ends with braces - if !strings.HasPrefix(cleanToken, "{") { - cleanToken = "{" + cleanToken - } - if !strings.HasSuffix(cleanToken, "}") { - cleanToken = cleanToken + "}" - } - // Remove all newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) + // Add read-only flag if specified + if readOnly != nil && *readOnly { + sectionContent += "read_only = true\n" } - configContent.WriteString(sourceSection) - configContent.WriteString("\n") + // Add start year if specified + if startYear > 0 { + sectionContent += fmt.Sprintf("start_year = %d\n", startYear) + } - case "sftp", "s3", "minio", "b2", "smb", "ftp", "webdav", "nextcloud", "onedrive": - // For complex source types, use the GenerateRcloneConfig function - // to create a temporary file, read it, and then append that content - tempDir, err := os.MkdirTemp("", "gomft_temp") + // Add include_archived flag if specified and true + if includeArchived != nil && *includeArchived { + sectionContent += "include_archived = true\n" + } + } + + // Find the section in the existing config + sectionPattern := regexp.MustCompile(fmt.Sprintf(`\[%s_%d\][^\[]*`, section, config.ID)) + if sectionPattern.MatchString(string(content)) { + // Replace the existing section + newContent := sectionPattern.ReplaceAllString(string(content), sectionContent) + err = os.WriteFile(configPath, []byte(newContent), 0644) if err != nil { - return fmt.Errorf("failed to create temp directory: %v", err) + return fmt.Errorf("failed to write updated config file: %v", err) } - defer os.RemoveAll(tempDir) - - tempConfigPath := filepath.Join(tempDir, "temp_config.conf") - - // Create a temporary file with just the source configuration - tempContent := fmt.Sprintf("[%s]\ntype = local\n", sourceName) - if err := os.WriteFile(tempConfigPath, []byte(tempContent), 0600); err != nil { - return fmt.Errorf("failed to write temporary config: %v", err) - } - - // Get rclone path - rclonePath := os.Getenv("RCLONE_PATH") - if rclonePath == "" { - rclonePath = "rclone" - } - - // Use the appropriate rclone command to configure the source - var args []string - switch config.SourceType { - case "sftp": - args = []string{ - "config", "create", sourceName, "sftp", - "host", config.SourceHost, - "user", config.SourceUser, - "port", fmt.Sprintf("%d", config.SourcePort), - "--non-interactive", - "--config", tempConfigPath, - "--log-level", "ERROR", - } - if config.SourcePassword != "" { - args = append(args, "pass", config.SourcePassword) - } - if config.SourceKeyFile != "" { - args = append(args, "key_file", config.SourceKeyFile) - } - case "s3": - args = []string{ - "config", "create", sourceName, "s3", - "provider", "AWS", - "env_auth", "false", - "access_key_id", config.SourceAccessKey, - "secret_access_key", config.SourceSecretKey, - "region", config.SourceRegion, - "--non-interactive", - "--config", tempConfigPath, - "--log-level", "ERROR", - } - if config.SourceEndpoint != "" { - args = append(args, "endpoint", config.SourceEndpoint) - } - } - - // If we have arguments, execute the command - if len(args) > 0 { - cmd := exec.Command(rclonePath, args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) - } - - // Read the generated config - sourceConfig, err := os.ReadFile(tempConfigPath) - if err != nil { - return fmt.Errorf("failed to read temporary config: %v", err) - } - - // Add it to our config content - configContent.WriteString(string(sourceConfig)) - configContent.WriteString("\n") - } - default: - // For local source or other simple types - sourceSection := fmt.Sprintf("[%s]\ntype = local\n\n", sourceName) - configContent.WriteString(sourceSection) - } - - // Now add the destination configuration - destName := fmt.Sprintf("dest_%d", config.ID) - - // Set up the destination section (only supporting Google Drive for now) - if config.DestinationType == "gdrive" || config.DestinationType == "google_drive" { - // Create a Google Drive remote using "dest_ID" naming convention - destSection := fmt.Sprintf("[%s]\ntype = drive\n", destName) - - // Add custom client ID and secret if provided - if config.DestClientID != "" && config.DestClientSecret != "" { - destSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n", - config.DestClientID, config.DestClientSecret) - } - - // Clean up token string to prevent syntax errors and ensure it's a single line JSON - // First, remove any whitespace from the beginning and end - cleanToken := strings.TrimSpace(token) - - // Check if it's already a JSON object - if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") { - // It's a JSON object, but we need to make sure it's a single line - var jsonObj map[string]interface{} - if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil { - // Successfully parsed the JSON, now re-marshal it as a compact single line - compactJSON, err := json.Marshal(jsonObj) - if err == nil { - // Use the compact JSON as the token - destSection += fmt.Sprintf("token = %s\n", string(compactJSON)) - } else { - // If there was an error re-marshaling, use the original but remove newlines - // Replace all newlines and carriage returns with empty string - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // If we couldn't parse the JSON, just remove newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // Not a valid JSON, try to fix it - // First ensure it starts and ends with braces - if !strings.HasPrefix(cleanToken, "{") { - cleanToken = "{" + cleanToken - } - if !strings.HasSuffix(cleanToken, "}") { - cleanToken = cleanToken + "}" - } - // Remove all newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - - // Add to the config content - configContent.WriteString(destSection) } else { - // Add a simple local destination for testing or if no specific destination type is handled - destSection := fmt.Sprintf("[%s]\ntype = local\n", destName) - configContent.WriteString(destSection) + // Append the section to the config + file, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open config file for appending: %v", err) + } + defer file.Close() + + _, err = file.WriteString("\n" + sectionContent) + if err != nil { + return fmt.Errorf("failed to append to config file: %v", err) + } } - // Write the config file - return os.WriteFile(configPath, []byte(configContent.String()), 0644) + // Update the authentication status + authenticated := true + if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" { + config.SetGoogleAuthenticated(authenticated) + } else if config.SourceType == "gdrive" || config.SourceType == "gphotos" { + config.SetGoogleAuthenticated(authenticated) + } + + return nil } // GetIsAdmin returns the value of IsAdmin with a default if nil @@ -1306,19 +1277,26 @@ func (tc *TransferConfig) SetDestPassiveMode(value bool) { tc.DestPassiveMode = &value } -// GetGoogleDriveAuthenticated returns the value of GoogleDriveAuthenticated with a default if nil +// GetGoogleDriveAuthenticated returns whether the transfer config has been authenticated with Google Drive func (tc *TransferConfig) GetGoogleDriveAuthenticated() bool { - if tc.GoogleDriveAuthenticated == nil { - return false // Default to false if not set - } - return *tc.GoogleDriveAuthenticated + return tc.GoogleDriveAuthenticated != nil && *tc.GoogleDriveAuthenticated } -// SetGoogleDriveAuthenticated sets the GoogleDriveAuthenticated field +// SetGoogleDriveAuthenticated sets the Google Drive authentication status func (tc *TransferConfig) SetGoogleDriveAuthenticated(value bool) { tc.GoogleDriveAuthenticated = &value } +// GetGoogleAuthenticated is an alias for GetGoogleDriveAuthenticated for better semantics when working with Google Photos +func (tc *TransferConfig) GetGoogleAuthenticated() bool { + return tc.GetGoogleDriveAuthenticated() +} + +// SetGoogleAuthenticated is an alias for SetGoogleDriveAuthenticated for better semantics when working with Google Photos +func (tc *TransferConfig) SetGoogleAuthenticated(value bool) { + tc.SetGoogleDriveAuthenticated(value) +} + // GetArchiveEnabled returns the value of ArchiveEnabled with a default if nil func (tc *TransferConfig) GetArchiveEnabled() bool { if tc.ArchiveEnabled == nil { diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 0495838..c1e605c 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -1416,3 +1416,345 @@ func TestGoogleDriveTeamDrive(t *testing.T) { err = db.DeleteTransferConfig(teamDriveBothConfig.ID) assert.NoError(t, err) } + +func TestGooglePhotosTransferConfig(t *testing.T) { + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-test-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err := db.CreateUser(testUser) + if err != nil { + t.Fatalf("Failed to create user: %v", err) + } + + // Test 1: Create config with Google Photos as source + sourceReadOnly := true + sourceIncludeArchived := false + useBuiltinAuth := true + googleSourceConfig := &TransferConfig{ + Name: "Google Photos Source Test", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "google_client_id", + SourceClientSecret: "google_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &sourceIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/local/destination/path", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Set authenticated status + authenticated := true + googleSourceConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleSourceConfig) + assert.NoError(t, err) + assert.NotZero(t, googleSourceConfig.ID, "Config ID should be set after creation") + + // Test 2: Create config with Google Photos as destination + destReadOnly := false + destIncludeArchived := true + googleDestConfig := &TransferConfig{ + Name: "Google Photos Destination Test", + SourceType: "local", + SourcePath: "/local/source/path", + DestinationType: "gphotos", + DestinationPath: "/albums/upload", + DestClientID: "google_client_id", + DestClientSecret: "google_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2018, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.png", + CreatedBy: testUser.ID, + } + + // Set authenticated status + googleDestConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleDestConfig) + assert.NoError(t, err) + assert.NotZero(t, googleDestConfig.ID, "Config ID should be set after creation") + + // Test 3: Create config with Google Photos as both source and destination + googleBothConfig := &TransferConfig{ + Name: "Google Photos Both Test", + SourceType: "gphotos", + SourcePath: "/albums/source_album", + SourceClientID: "source_client_id", + SourceClientSecret: "source_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2020, + SourceIncludeArchived: &sourceIncludeArchived, + DestinationType: "gphotos", + DestinationPath: "/albums/dest_album", + DestClientID: "dest_client_id", + DestClientSecret: "dest_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2020, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.jpeg", + CreatedBy: testUser.ID, + } + + // Set authenticated status + googleBothConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleBothConfig) + assert.NoError(t, err) + assert.NotZero(t, googleBothConfig.ID, "Config ID should be set after creation") + + // Verify configs were created properly + retrievedConfig, err := db.GetTransferConfig(googleSourceConfig.ID) + assert.NoError(t, err) + assert.Equal(t, "gphotos", retrievedConfig.SourceType) + assert.Equal(t, sourceReadOnly, *retrievedConfig.SourceReadOnly) + assert.Equal(t, 2015, retrievedConfig.SourceStartYear) + assert.Equal(t, sourceIncludeArchived, *retrievedConfig.SourceIncludeArchived) + assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth) + assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated()) + + retrievedConfig, err = db.GetTransferConfig(googleDestConfig.ID) + assert.NoError(t, err) + assert.Equal(t, "gphotos", retrievedConfig.DestinationType) + assert.Equal(t, destReadOnly, *retrievedConfig.DestReadOnly) + assert.Equal(t, 2018, retrievedConfig.DestStartYear) + assert.Equal(t, destIncludeArchived, *retrievedConfig.DestIncludeArchived) + assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth) + assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated()) +} + +func TestGooglePhotosRcloneConfig(t *testing.T) { + // Create a temporary test directory + tempDir, err := os.MkdirTemp("", "gomft-test") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Set up data directory + dataDir := filepath.Join(tempDir, "data") + configDir := filepath.Join(dataDir, "configs") + err = os.MkdirAll(configDir, 0755) + if err != nil { + t.Fatalf("Failed to create config directory: %v", err) + } + + // Set DATA_DIR environment variable + oldDataDir := os.Getenv("DATA_DIR") + defer os.Setenv("DATA_DIR", oldDataDir) + os.Setenv("DATA_DIR", dataDir) + + // Initialize test database + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-rclone-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err = db.CreateUser(testUser) + assert.NoError(t, err) + + // TEST 1: Google Photos as source with standard options + sourceReadOnly := true + sourceIncludeArchived := false + useBuiltinAuth := true + gphotosSourceConfig := &TransferConfig{ + ID: 1, // Force ID for predictable config path + Name: "Google Photos Source Config", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "test_client_id", + SourceClientSecret: "test_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &sourceIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/tmp/destination", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Generate rclone config + err = db.GenerateRcloneConfig(gphotosSourceConfig) + assert.NoError(t, err) + + // Check if config file exists + configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosSourceConfig.ID)) + _, err = os.Stat(configPath) + assert.NoError(t, err, "Config file should exist") + + // Read config file content + content, err := os.ReadFile(configPath) + assert.NoError(t, err) + configContent := string(content) + + // Check for Google Photos source section + assert.Contains(t, configContent, "[source_1]") + assert.Contains(t, configContent, "type = google photos") + assert.Contains(t, configContent, "client_id = test_client_id") + assert.Contains(t, configContent, "client_secret = test_client_secret") + assert.Contains(t, configContent, "read_only = true") + assert.Contains(t, configContent, "start_year = 2015") + assert.NotContains(t, configContent, "include_archived = true") // This should be false and not included + + // TEST 2: Google Photos as destination with authenticated token + destReadOnly := false + destIncludeArchived := true + gphotosDestConfig := &TransferConfig{ + ID: 2, // Force ID for predictable config path + Name: "Google Photos Destination Config", + SourceType: "local", + SourcePath: "/tmp/source", + DestinationType: "gphotos", + DestinationPath: "/albums/upload", + DestClientID: "dest_client_id", + DestClientSecret: "dest_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2018, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.png", + CreatedBy: testUser.ID, + } + + // Set authentication status + authenticated := true + gphotosDestConfig.GoogleDriveAuthenticated = &authenticated + + // Generate config first (needed for token update) + err = db.GenerateRcloneConfig(gphotosDestConfig) + assert.NoError(t, err) + + // Now test token handling with GenerateRcloneConfigWithToken + testToken := `{"access_token":"test-token","token_type":"Bearer","refresh_token":"test-refresh","expiry":"2023-12-31T23:59:59Z"}` + err = db.GenerateRcloneConfigWithToken(gphotosDestConfig, testToken) + assert.NoError(t, err) + + // Check updated config + configPath = filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosDestConfig.ID)) + _, err = os.Stat(configPath) + assert.NoError(t, err, "Config file should exist") + + // Read config file content + content, err = os.ReadFile(configPath) + assert.NoError(t, err) + configContent = string(content) + + // Check for Google Photos destination section with token + assert.Contains(t, configContent, "type = google photos") + assert.Contains(t, configContent, "client_id = dest_client_id") + assert.Contains(t, configContent, "client_secret = dest_client_secret") + assert.Contains(t, configContent, "token = {") + assert.Contains(t, configContent, "access_token") + assert.Contains(t, configContent, "test-token") + assert.Contains(t, configContent, "refresh_token") + assert.Contains(t, configContent, "test-refresh") + assert.Contains(t, configContent, "include_archived = true") + assert.NotContains(t, configContent, "read_only = false") // This should be false and not included +} + +func TestGooglePhotosAuthentication(t *testing.T) { + // Initialize test database + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-auth-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err := db.CreateUser(testUser) + assert.NoError(t, err) + + // Create a transfer config with Google Photos + readOnly := true + includeArchived := false + useBuiltinAuth := true + gPhotosConfig := &TransferConfig{ + Name: "Test Google Photos Auth", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "test_client_id", + SourceClientSecret: "test_client_secret", + SourceReadOnly: &readOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &includeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/tmp/destination", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Create the config + err = db.CreateTransferConfig(gPhotosConfig) + assert.NoError(t, err) + assert.NotZero(t, gPhotosConfig.ID) + + // Test initial authentication state + // Should be false when first created + authenticated := gPhotosConfig.GetGoogleAuthenticated() + assert.False(t, authenticated) + t.Logf("Initial GoogleDriveAuthenticated value: %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Test generic Google authentication method (new) + gPhotosConfig.SetGoogleAuthenticated(true) + t.Logf("After SetGoogleAuthenticated(true): %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(gPhotosConfig) + assert.NoError(t, err) + t.Logf("After UpdateTransferConfig: %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Verify authentication status is updated + updatedConfig, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config GoogleDriveAuthenticated: %v", updatedConfig.GoogleDriveAuthenticated) + assert.True(t, updatedConfig.GetGoogleAuthenticated()) + + // Verify it can be unset + updatedConfig.SetGoogleAuthenticated(false) + t.Logf("After SetGoogleAuthenticated(false): %v", updatedConfig.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(updatedConfig) + assert.NoError(t, err) + + // Verify authentication status is updated + updatedConfig2, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config2 GoogleDriveAuthenticated: %v", updatedConfig2.GoogleDriveAuthenticated) + assert.False(t, updatedConfig2.GetGoogleAuthenticated()) + + // Test with the old method naming for backward compatibility + updatedConfig2.SetGoogleDriveAuthenticated(true) + t.Logf("After SetGoogleDriveAuthenticated(true): %v", updatedConfig2.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(updatedConfig2) + assert.NoError(t, err) + + // Verify authentication status is updated when using the old method + updatedConfig3, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config3 GoogleDriveAuthenticated: %v", updatedConfig3.GoogleDriveAuthenticated) + assert.True(t, updatedConfig3.GetGoogleAuthenticated()) + assert.True(t, updatedConfig3.GetGoogleDriveAuthenticated()) +} diff --git a/internal/db/migrations/add_google_photos_support.go b/internal/db/migrations/add_google_photos_support.go new file mode 100644 index 0000000..2e2255d --- /dev/null +++ b/internal/db/migrations/add_google_photos_support.go @@ -0,0 +1,61 @@ +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddGooglePhotosSupport adds Google Photos related fields to the transfer_configs table +func AddGooglePhotosSupport() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20240518_add_google_photos_support", + Migrate: func(tx *gorm.DB) error { + // Add Google Photos source fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_read_only BOOLEAN DEFAULT false").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_start_year INTEGER DEFAULT 0").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_include_archived BOOLEAN DEFAULT false").Error; err != nil { + return err + } + + // Add Google Photos destination fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_read_only BOOLEAN DEFAULT false").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_start_year INTEGER DEFAULT 0").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_include_archived BOOLEAN DEFAULT false").Error; err != nil { + return err + } + + // Add OAuth field + return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN use_builtin_auth BOOLEAN DEFAULT true").Error + }, + Rollback: func(tx *gorm.DB) error { + // Remove all added columns in reverse order + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN use_builtin_auth").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_include_archived").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_start_year").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_read_only").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_include_archived").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_start_year").Error; err != nil { + return err + } + return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_read_only").Error + }, + } +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index b0953f4..fea061e 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -17,6 +17,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate { UpdateSkipProcessedFilesToNullable(), AddWebhookSupport(), AddGoogleDriveAuthenticated(), + AddGooglePhotosSupport(), } return gormigrate.New(db, gormigrate.DefaultOptions, migrations) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 6f4b615..61da064 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -535,6 +535,12 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, if maxConcurrent < 1 { maxConcurrent = 1 // Default to 1 if not set } + + // Limit Google Photos to 1 concurrent transfers + if config.SourceType == "gphotos" || config.DestinationType == "gphotos" { + maxConcurrent = 1 + } + s.log.LogInfo("Using %d concurrent transfers for job %d, config %d", maxConcurrent, job.ID, config.ID) // Create wait group for concurrent processing diff --git a/internal/web/handlers/config_handlers.go b/internal/web/handlers/config_handlers.go index 42241f1..dc886a5 100644 --- a/internal/web/handlers/config_handlers.go +++ b/internal/web/handlers/config_handlers.go @@ -101,6 +101,27 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) { destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true" config.DestPassiveMode = &destPassiveModeValue + // Google Photos specific fields + destReadOnlyVal := c.Request.FormValue("dest_read_only") + destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true" + config.DestReadOnly = &destReadOnlyValue + + sourceReadOnlyVal := c.Request.FormValue("source_read_only") + sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true" + config.SourceReadOnly = &sourceReadOnlyValue + + destIncludeArchivedVal := c.Request.FormValue("dest_include_archived") + destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true" + config.DestIncludeArchived = &destIncludeArchivedValue + + sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived") + sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" + config.SourceIncludeArchived = &sourceIncludeArchivedValue + + useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth") + useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true" + config.UseBuiltinAuth = &useBuiltinAuthValue + if err := h.DB.Create(&config).Error; err != nil { log.Printf("Error creating config: %v", err) c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to create config: %v", err)) @@ -171,6 +192,27 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) { destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true" config.DestPassiveMode = &destPassiveModeValue + // Google Photos specific fields + destReadOnlyVal := c.Request.FormValue("dest_read_only") + destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true" + config.DestReadOnly = &destReadOnlyValue + + sourceReadOnlyVal := c.Request.FormValue("source_read_only") + sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true" + config.SourceReadOnly = &sourceReadOnlyValue + + destIncludeArchivedVal := c.Request.FormValue("dest_include_archived") + destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true" + config.DestIncludeArchived = &destIncludeArchivedValue + + sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived") + sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" + config.SourceIncludeArchived = &sourceIncludeArchivedValue + + useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth") + useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true" + config.UseBuiltinAuth = &useBuiltinAuthValue + // Preserve fields that shouldn't be updated config.CreatedBy = oldConfig.CreatedBy diff --git a/internal/web/handlers/gdrive_handlers.go b/internal/web/handlers/gdrive_handlers.go index 0f4ced4..96ed2c1 100644 --- a/internal/web/handlers/gdrive_handlers.go +++ b/internal/web/handlers/gdrive_handlers.go @@ -36,9 +36,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { return } - // Ensure it's a Google Drive configuration - if config.DestinationType != "gdrive" { - RenderErrorPage(c, "Not a Google Drive configuration", "The selected configuration is not set up for Google Drive") + // Ensure it's a Google Drive or Google Photos configuration + if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" { + RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos") return } @@ -81,9 +81,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { // Define the redirect URI for our callback redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL) - // Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV - clientID := os.Getenv("GDRIVE_CLIENT_ID") - clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET") + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") if clientID == "" || clientSecret == "" { // Check if we have client credentials in the existing config file @@ -109,7 +109,7 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { clientSecret = existingClientSecret } else { // If we still can't find a matching secret, show an error - RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.") + RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.") return } } @@ -123,13 +123,28 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { // Store config ID in cookie for use during callback c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true) + // Determine the appropriate scope based on destination type + var scope string + if config.DestinationType == "gphotos" { + // Read-only access is handled elsewhere in the config; here we need the full auth scope + scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary") + } else { + // Default to Google Drive scope + scope = url.QueryEscape("https://www.googleapis.com/auth/drive") + } + // Create a config file with redirect URI-based auth - configContent := fmt.Sprintf(`[temp_gdrive] -type = drive + configType := "drive" + if config.DestinationType == "gphotos" { + configType = "google photos" + } + + configContent := fmt.Sprintf(`[temp_%s] +type = %s client_id = %s client_secret = %s redirect_url = %s -`, clientID, clientSecret, redirectURI) +`, config.DestinationType, configType, clientID, clientSecret, redirectURI) // Write the config file if err := os.WriteFile(tempConfigPath, []byte(configContent), 0644); err != nil { @@ -138,7 +153,6 @@ redirect_url = %s } // Direct Google OAuth URL with our redirect - scope := url.QueryEscape("https://www.googleapis.com/auth/drive") authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s", url.QueryEscape(clientID), url.QueryEscape(redirectURI), @@ -205,9 +219,9 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { return } - // Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV - clientID := os.Getenv("GDRIVE_CLIENT_ID") - clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET") + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") if clientID == "" || clientSecret == "" { // Check if we have client credentials in the existing config file @@ -233,7 +247,7 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { clientSecret = existingClientSecret } else { // If we still can't find a matching secret, show an error - RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.") + RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.") return } } @@ -313,7 +327,13 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { c.SetCookie("gdrive_config_id", "", -1, "/", "", false, true) // Redirect to the config list with a success message - c.Redirect(http.StatusFound, "/configs?status=gdrive_auth_success") + var successParam string + if config.DestinationType == "gphotos" { + successParam = "gphotos_auth_success" + } else { + successParam = "gdrive_auth_success" + } + c.Redirect(http.StatusFound, fmt.Sprintf("/configs?status=%s", successParam)) } // HandleGDriveTokenProcess processes a Google Drive token directly from a URL parameter diff --git a/internal/web/handlers/gdrive_handlers_test.go b/internal/web/handlers/gdrive_handlers_test.go new file mode 100644 index 0000000..30d507f --- /dev/null +++ b/internal/web/handlers/gdrive_handlers_test.go @@ -0,0 +1,429 @@ +package handlers + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/starfleetcptn/gomft/internal/db" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// DBInterface defines the methods we need to mock for our tests +type DBInterface interface { + GetTransferConfig(id uint) (*db.TransferConfig, error) + GetConfigRclonePath(config *db.TransferConfig) string + GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error + GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string) +} + +// MockDB is a mock implementation of the DB interface for testing +type MockDB struct { + mock.Mock +} + +// Implement the necessary methods from the DB interface for our tests +func (m *MockDB) GetTransferConfig(id uint) (*db.TransferConfig, error) { + args := m.Called(id) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*db.TransferConfig), args.Error(1) +} + +func (m *MockDB) GetConfigRclonePath(config *db.TransferConfig) string { + args := m.Called(config) + return args.String(0) +} + +func (m *MockDB) GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error { + args := m.Called(config, token) + return args.Error(0) +} + +func (m *MockDB) GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string) { + args := m.Called(config) + return args.String(0), args.String(1) +} + +// MockHandlers is a modified version of Handlers that accepts our mock DB +type MockHandlers struct { + DB DBInterface +} + +// HandleGDriveAuth is a copy of the original method but using our interface +func (h *MockHandlers) HandleGDriveAuth(c *gin.Context) { + // Get the config ID from the query parameter + configIDStr := c.Param("id") + if configIDStr == "" { + RenderErrorPage(c, "Missing configuration ID", "") + return + } + + configID, err := strconv.ParseUint(configIDStr, 10, 64) + if err != nil { + RenderErrorPage(c, "Invalid configuration ID", err.Error()) + return + } + + // Get the configuration + config, err := h.DB.GetTransferConfig(uint(configID)) + if err != nil { + RenderErrorPage(c, "Configuration not found", err.Error()) + return + } + + // Ensure it's a Google Drive or Google Photos configuration + if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" { + RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos") + return + } + + // Prepare for OAuth + dataDir := os.Getenv("DATA_DIR") + if dataDir == "" { + dataDir = "./data" + } + + // Get Rclone Config Path + rcloneConfigPath := h.DB.GetConfigRclonePath(config) + if rcloneConfigPath == "" { + RenderErrorPage(c, "Rclone config not found", "The selected configuration does not have a valid rclone config") + return + } + + // Create a temporary config file for authentication + tempConfigDir := filepath.Join(dataDir, "temp") + if err := os.MkdirAll(tempConfigDir, 0755); err != nil { + RenderErrorPage(c, "Failed to create temporary directory", err.Error()) + return + } + + tempConfigPath := filepath.Join(tempConfigDir, fmt.Sprintf("gdrive_auth_%d.conf", config.ID)) + + // Store the temporary config path in a cookie + c.SetCookie("gdrive_temp_config", tempConfigPath, 3600, "/", "", false, true) + + // Get base URL for redirect URI + baseURL := os.Getenv("BASE_URL") + if baseURL == "" { + // Try to detect the base URL from the request + scheme := "http" + if c.Request.TLS != nil { + scheme = "https" + } + baseURL = fmt.Sprintf("%s://%s", scheme, c.Request.Host) + } + + // Define the redirect URI for our callback + redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL) + + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") + + if clientID == "" || clientSecret == "" { + // Check if we have client credentials in the existing config file + existingClientID, existingClientSecret := h.DB.GetGDriveCredentialsFromConfig(config) + + if existingClientID != "" && existingClientSecret != "" { + // Use credentials from existing config + clientID = existingClientID + clientSecret = existingClientSecret + } else { + // fallback to rclone client ID and secret + clientID = "202264815644.apps.googleusercontent.com" + clientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ" + } + } + + // Generate state parameter for security (to prevent CSRF) + state := fmt.Sprintf("gomft_%d_%d", config.ID, time.Now().Unix()) + + // Store state in cookie for validation during callback + c.SetCookie("gdrive_auth_state", state, 3600, "/", "", false, true) + + // Store config ID in cookie for use during callback + c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true) + + // Determine the appropriate scope based on destination type + var scope string + if config.DestinationType == "gphotos" { + // Read-only access is handled elsewhere in the config; here we need the full auth scope + scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary") + } else { + // Default to Google Drive scope + scope = url.QueryEscape("https://www.googleapis.com/auth/drive") + } + + // Direct Google OAuth URL with our redirect + authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s", + url.QueryEscape(clientID), + url.QueryEscape(redirectURI), + scope, + url.QueryEscape(state)) + + // Redirect the user to Google's auth page directly + c.Redirect(http.StatusFound, authURL) +} + +// HandleGDriveAuthCallback handles the callback from Google OAuth +func (h *MockHandlers) HandleGDriveAuthCallback(c *gin.Context) { + // Get auth code from query parameters + authCode := c.Query("code") + if authCode == "" { + RenderErrorPage(c, "Authentication failed", "No authorization code received from Google") + return + } + + // Verify state parameter to prevent CSRF + state := c.Query("state") + storedState, err := c.Cookie("gdrive_auth_state") + if err != nil || state != storedState { + RenderErrorPage(c, "Authentication failed", "Invalid state parameter") + return + } + + // Get config ID from cookie + configIDStr, err := c.Cookie("gdrive_config_id") + if err != nil { + RenderErrorPage(c, "Authentication failed", "Unable to retrieve configuration ID") + return + } + + configID, err := strconv.ParseUint(configIDStr, 10, 64) + if err != nil { + RenderErrorPage(c, "Invalid configuration ID", err.Error()) + return + } + + // Get the configuration + config, err := h.DB.GetTransferConfig(uint(configID)) + if err != nil { + RenderErrorPage(c, "Failed to get configuration", err.Error()) + return + } + + // For testing purposes, we'll simulate a successful token exchange + // In a real implementation, we would exchange the auth code for a token + mockToken := `{"access_token":"test_access_token","refresh_token":"test_refresh_token","expiry":"2023-12-31T23:59:59Z"}` + + // Update the config with the token + err = h.DB.GenerateRcloneConfigWithToken(config, mockToken) + if err != nil { + RenderErrorPage(c, "Failed to update configuration", err.Error()) + return + } + + // Redirect to the config edit page + c.Redirect(http.StatusFound, fmt.Sprintf("/configs/edit/%d", config.ID)) +} + +func setupTestRouter() (*gin.Engine, *MockDB) { + gin.SetMode(gin.TestMode) + router := gin.New() + mockDB := new(MockDB) + handlers := &MockHandlers{ + DB: mockDB, + } + + router.GET("/configs/gdrive/:id", handlers.HandleGDriveAuth) + router.GET("/configs/gdrive-callback", handlers.HandleGDriveAuthCallback) + + return router, mockDB +} + +func TestHandleGDriveAuth_GoogleDrive(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 1, + DestinationType: "gdrive", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil) + mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf") + mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret") + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/1", nil) + router.ServeHTTP(w, req) + + // Assertions + assert.Equal(t, http.StatusFound, w.Code) + + // Verify the redirect URL + location := w.Header().Get("Location") + assert.Contains(t, location, "accounts.google.com/o/oauth2/auth") + assert.Contains(t, location, "drive") + assert.Contains(t, location, "test_client_id") + + // Verify cookies were set + cookies := w.Result().Cookies() + assert.GreaterOrEqual(t, len(cookies), 3) + + // Check if state cookie exists + stateFound := false + for _, cookie := range cookies { + if cookie.Name == "gdrive_auth_state" { + stateFound = true + break + } + } + assert.True(t, stateFound) +} + +func TestHandleGDriveAuth_GooglePhotos(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 2, + DestinationType: "gphotos", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(2)).Return(testConfig, nil) + mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf") + mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret") + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/2", nil) + router.ServeHTTP(w, req) + + // Assertions + assert.Equal(t, http.StatusFound, w.Code) + + // Verify the redirect URL + location := w.Header().Get("Location") + assert.Contains(t, location, "accounts.google.com/o/oauth2/auth") + assert.Contains(t, location, "photoslibrary") + assert.Contains(t, location, "test_client_id") + + // Verify cookies were set + cookies := w.Result().Cookies() + assert.GreaterOrEqual(t, len(cookies), 3) + + // Check if state cookie exists + stateFound := false + for _, cookie := range cookies { + if cookie.Name == "gdrive_auth_state" { + stateFound = true + break + } + } + assert.True(t, stateFound) +} + +func TestHandleGDriveAuthCallback(t *testing.T) { + // Setup test environment + router, mockDB := setupTestRouter() + + // Create a temporary directory for testing + tempDir, err := os.MkdirTemp("", "gdrive-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create a temporary config file + tempConfigPath := filepath.Join(tempDir, "temp_config.conf") + if err := os.WriteFile(tempConfigPath, []byte("test config"), 0644); err != nil { + t.Fatal(err) + } + + // Test state and config ID + testState := "gomft_1_12345" + testConfigID := "1" + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 1, + DestinationType: "gphotos", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil) + mockDB.On("GenerateRcloneConfigWithToken", testConfig, mock.Anything).Return(nil) + + // Create test request with auth code and state + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive-callback?code=test_auth_code&state="+testState, nil) + + // Add required cookies to the request + req.AddCookie(&http.Cookie{Name: "gdrive_auth_state", Value: testState}) + req.AddCookie(&http.Cookie{Name: "gdrive_config_id", Value: testConfigID}) + req.AddCookie(&http.Cookie{Name: "gdrive_temp_config", Value: tempConfigPath}) + + // Send the request + router.ServeHTTP(w, req) + + // We expect a redirect on successful auth + assert.Equal(t, http.StatusFound, w.Code) + + // Should redirect to the config edit page + location := w.Header().Get("Location") + assert.Contains(t, location, "/configs/edit/1") +} + +func TestHandleGDriveAuth_InvalidConfig(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Set up mock expectations for a non-existent config + mockDB.On("GetTransferConfig", uint(999)).Return(nil, fmt.Errorf("config not found")) + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/999", nil) + router.ServeHTTP(w, req) + + // Assertions - should render error page + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "Configuration not found") +} + +func TestHandleGDriveAuth_NonGoogleConfig(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a non-Google test config + testConfig := &db.TransferConfig{ + ID: 3, + DestinationType: "s3", // Not Google Drive or Photos + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(3)).Return(testConfig, nil) + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/3", nil) + router.ServeHTTP(w, req) + + // Assertions - should render error page + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "Not a Google configuration") +} + +// RenderErrorPage renders an error page with the given message +func RenderErrorPage(c *gin.Context, title string, details string) { + // Here we'd typically use a component for error display + // For now, we'll just render a simple HTML error page for testing + errorHTML := fmt.Sprintf("

Error: %s

%s

", title, details) + c.Data(http.StatusOK, "text/html", []byte(errorHTML)) +} From f1fd33c6191aa9fefeaa70696fe8bbe230052fa2 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sat, 15 Mar 2025 23:42:24 -0700 Subject: [PATCH 3/3] feat: Enhance README with Google Drive and Google Photos configuration details - Add Google Drive and Google Photos as supported storage options in the features section. - Include Google OAuth configuration parameters in the environment variable section. - Update transfer configuration options to specify Google Photos and Google Drive specific settings. - Improve documentation clarity for file options and performance settings. --- README.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ea2fe60..0a65571 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging ## Features - **Multiple Storage Support**: Leverage rclone's extensive support for cloud storage providers: + - Google Drive + - Google Photos - Amazon S3 - MinIO - NextCloud @@ -150,6 +152,10 @@ services: - BACKUP_DIR=/app/backups - JWT_SECRET=change_this_to_a_secure_random_string - BASE_URL=http://localhost:8080 + # Google OAuth configuration (optional) + - GOOGLE_CLIENT_ID=your_google_client_id + - GOOGLE_CLIENT_SECRET=your_google_client_secret + # Email configuration - EMAIL_ENABLED=true - EMAIL_HOST=smtp.example.com - EMAIL_PORT=587 @@ -206,6 +212,10 @@ BACKUP_DIR=/app/backups JWT_SECRET=change_this_to_a_secure_random_string BASE_URL=http://localhost:8080 +# Google OAuth configuration (optional, for built-in authentication) +GOOGLE_CLIENT_ID=your_google_client_id +GOOGLE_CLIENT_SECRET=your_google_client_secret + # Email configuration EMAIL_ENABLED=true EMAIL_HOST=smtp.example.com @@ -226,6 +236,9 @@ EMAIL_PASSWORD=smtp_password - `BACKUP_DIR`: Directory for storing database backups - `JWT_SECRET`: Secret key for JWT token generation - `BASE_URL`: Base URL for generating links in emails (e.g., password reset links) +- Google OAuth configuration for built-in authentication: + - `GOOGLE_CLIENT_ID`: Your Google OAuth client ID + - `GOOGLE_CLIENT_SECRET`: Your Google OAuth client secret - Email configuration settings for system notifications and password resets: - `EMAIL_ENABLED`: Set to `true` to enable email functionality - `EMAIL_HOST`: SMTP server hostname @@ -332,6 +345,8 @@ User management features: ### Transfer Configuration Options 1. **Source/Destination Types**: + - Google Drive (with built-in or custom authentication) + - Google Photos (with built-in or custom authentication) - Local filesystem - Amazon S3 - MinIO (S3-compatible storage) @@ -343,32 +358,46 @@ User management features: 2. **Connection Options**: - Host/server addresses - Authentication (username/password or key files) + - OAuth2 authentication for Google services - Port configurations - Cloud credentials (access keys, secret keys) - Bucket and region settings - Custom endpoints - Custom rclone flags -3. **File Options**: +3. **Google Photos Specific Options**: + - Read-only mode for safer operations + - Start year filter for historical photos + - Include/exclude archived media + - Album path configuration + - Built-in or custom OAuth authentication + +4. **Google Drive Specific Options**: + - Folder ID for specific directory access + - Team/Shared Drive ID support + - Built-in or custom OAuth authentication + - Path-based navigation + +5. **File Options**: - File patterns for filtering (e.g., `*.txt`, `data_*.csv`) - Output patterns for dynamic naming - Archive options for transferred files - Skip already processed files to avoid duplicates - Concurrent file transfers (configurable per job) -4. **Performance Options**: +6. **Performance Options**: - **Multi-threaded File Transfers**: Process multiple files simultaneously for higher throughput - Configurable concurrency level (1-32 concurrent transfers) - Per-job concurrency settings to optimize for different storage types - Automatic transfer queue management to prevent overloading systems - Adaptive processing based on source/destination capabilities -5. **Schedule Options**: +7. **Schedule Options**: - Cron expressions for flexible scheduling - Manual execution - Enable/disable schedules -6. **Webhook Notifications**: +8. **Webhook Notifications**: - **Webhook Integration**: Send notifications to external systems when jobs complete - **Secure Authentication**: HMAC-SHA256 signature for webhook verification - **Custom Headers**: Add custom HTTP headers to webhook requests