From d68a80bd784821dcccb6c823f0f75bab01cf46cf Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Sat, 15 Mar 2025 23:16:49 -0700 Subject: [PATCH] feat: Add Google Photos support for transfer configurations - Introduce Google Photos as a source and destination option in the configuration forms. - Implement authentication handling and configuration management for Google Photos. - Update database schema and migrations to include new fields related to Google Photos. - Enhance UI components to support Google Photos-specific settings and forms. - Add tests for Google Photos integration, ensuring proper functionality and authentication flow. - Limit concurrent transfers for Google Photos to ensure compliance with API restrictions. --- components/config_form.templ | 47 +- components/configs.templ | 6 +- components/providers/common/common.templ | 2 + .../providers/destination/gphotos.templ | 220 +++++++++ components/providers/source/gphotos.templ | 220 +++++++++ internal/db/db.go | 414 ++++++++--------- internal/db/db_test.go | 342 ++++++++++++++ .../migrations/add_google_photos_support.go | 61 +++ internal/db/migrations/migrations.go | 1 + internal/scheduler/scheduler.go | 6 + internal/web/handlers/config_handlers.go | 42 ++ internal/web/handlers/gdrive_handlers.go | 52 ++- internal/web/handlers/gdrive_handlers_test.go | 429 ++++++++++++++++++ 13 files changed, 1604 insertions(+), 238 deletions(-) create mode 100644 components/providers/destination/gphotos.templ create mode 100644 components/providers/source/gphotos.templ create mode 100644 internal/db/migrations/add_google_photos_support.go create mode 100644 internal/web/handlers/gdrive_handlers_test.go diff --git a/components/config_form.templ b/components/config_form.templ index d392ba8..6bb6ddc 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -7,6 +7,7 @@ import ( "github.com/starfleetcptn/gomft/components/providers/source" "github.com/starfleetcptn/gomft/components/providers/destination" "github.com/starfleetcptn/gomft/components/providers/common" + "strconv" ) type ConfigFormData struct { @@ -44,6 +45,10 @@ func getInitialData(config *db.TransferConfig) string { sourceClientSecret := "" sourceDriveId := "" sourceTeamDrive := "" + // Google Photos source fields + sourceReadOnly := false + sourceStartYear, _ := strconv.Atoi(getCurrentYear()) + sourceIncludeArchived := false filePattern := "" outputPattern := "${filename}" @@ -68,6 +73,10 @@ func getInitialData(config *db.TransferConfig) string { destClientSecret := "" destDriveId := "" destTeamDrive := "" + // Google Photos destination fields + destReadOnly := false + destStartYear, _ := strconv.Atoi(getCurrentYear()) // Default to current year int + destIncludeArchived := false archivePath := "" archiveEnabled := false @@ -103,6 +112,15 @@ func getInitialData(config *db.TransferConfig) string { sourceDriveId = config.SourceDriveID sourceTeamDrive = config.SourceTeamDrive + // Google Photos source fields + if config.SourceReadOnly != nil { + sourceReadOnly = *config.SourceReadOnly + } + sourceStartYear = config.SourceStartYear + if config.SourceIncludeArchived != nil { + sourceIncludeArchived = *config.SourceIncludeArchived + } + filePattern = config.FilePattern outputPattern = config.OutputPattern @@ -129,13 +147,24 @@ func getInitialData(config *db.TransferConfig) string { destDriveId = config.DestDriveID destTeamDrive = config.DestTeamDrive + // Google Photos destination fields + if config.DestReadOnly != nil { + destReadOnly = *config.DestReadOnly + } + destStartYear = config.DestStartYear + if config.DestIncludeArchived != nil { + destIncludeArchived = *config.DestIncludeArchived + } + archivePath = config.ArchivePath archiveEnabled = config.GetArchiveEnabled() deleteAfterTransfer = config.GetDeleteAfterTransfer() skipProcessedFiles = config.GetSkipProcessedFiles() maxConcurrentTransfers = config.MaxConcurrentTransfers rcloneFlags = config.RcloneFlags - if destClientId != "" || destClientSecret != "" { + if config.UseBuiltinAuth != nil { + useBuiltinAuth = *config.UseBuiltinAuth + } else if destClientId != "" || destClientSecret != "" { useBuiltinAuth = false } } @@ -163,6 +192,9 @@ func getInitialData(config *db.TransferConfig) string { sourceClientSecret: '%s', sourceDriveId: '%s', sourceTeamDrive: '%s', + sourceReadOnly: %v, + sourceStartYear: %d, + sourceIncludeArchived: %v, filePattern: '%s', outputPattern: '%s', @@ -187,6 +219,10 @@ func getInitialData(config *db.TransferConfig) string { destClientSecret: '%s', destDriveId: '%s', destTeamDrive: '%s', + destReadOnly: %v, + destStartYear: %d, + destIncludeArchived: %v, + useBuiltinAuth: %v, archivePath: '%s', @@ -199,10 +235,12 @@ func getInitialData(config *db.TransferConfig) string { name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType, sourceBucket, sourceRegion, sourceAccessKey, sourceSecretKey, sourceEndpoint, sourceShare, sourceDomain, sourcePassiveMode, sourceClientId, sourceClientSecret, sourceDriveId, sourceTeamDrive, + sourceReadOnly, sourceStartYear, sourceIncludeArchived, filePattern, outputPattern, destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType, destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode, destClientId, destClientSecret, destDriveId, destTeamDrive, + destReadOnly, destStartYear, destIncludeArchived, useBuiltinAuth, archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags) } @@ -287,6 +325,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @source.GoogleDriveSourceForm() + + @common.FilePatternFields() @@ -330,6 +372,9 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @destination.GoogleDriveDestinationForm() + @common.ArchiveOptions() diff --git a/components/configs.templ b/components/configs.templ index a15ded9..49d4597 100644 --- a/components/configs.templ +++ b/components/configs.templ @@ -270,7 +270,7 @@ templ Configs(ctx context.Context, data ConfigsData) {

- if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && !config.GetGoogleDriveAuthenticated() { + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && !config.GetGoogleAuthenticated() { Authentication Required @@ -278,7 +278,7 @@ templ Configs(ctx context.Context, data ConfigsData) { } - if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && config.GetGoogleDriveAuthenticated() { + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && config.GetGoogleAuthenticated() { Authenticated @@ -287,7 +287,7 @@ templ Configs(ctx context.Context, data ConfigsData) {
- if (config.DestinationType == "gdrive" || config.SourceType == "gdrive") && !config.GetGoogleDriveAuthenticated() { + if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && !config.GetGoogleAuthenticated() { Authenticate diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index 0ffddb5..8562ee2 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -195,6 +195,7 @@ templ SourceSelection() { +
@@ -219,6 +220,7 @@ templ DestinationSelection() { + diff --git a/components/providers/destination/gphotos.templ b/components/providers/destination/gphotos.templ new file mode 100644 index 0000000..7fc56f7 --- /dev/null +++ b/components/providers/destination/gphotos.templ @@ -0,0 +1,220 @@ +package destination + +templ GooglePhotosDestinationForm() { +
+
+ +

+ Simple one-click authentication using rclone's shared credentials +

+
+ +
+
+ +
+
+ +
+ +
+

+ Client ID from Google Cloud Console +

+
+ +
+ +
+
+ +
+ +
+

+ Client Secret from Google Cloud Console +

+
+
+ +
+ +

+ Only request read-only access to your photos +

+
+ +
+ +
+
+ +
+ +
+

+ Only include photos uploaded after this year +

+
+ +
+ +

+ Include archived photos and videos in media listings +

+
+ +
+ +
+
+ +
+ +
+

+ Path within Google Photos where files will be uploaded +

+
+ +
+
+
+ +
+
+

Important: Authentication Required

+
+

After saving this configuration, you will need to authenticate with Google Photos.

+

The authentication process will require you to:

+
    +
  1. Visit a Google authorization URL
    2. +
  2. Sign in to your Google account
    3. +
  3. Grant permission to access your Google Photos
    4. +
  4. Copy the authorization code back to this application
    5. +
+

+ This is a one-time process for each configuration. The application will store your authorization token securely. +

+
+
+
+
+ +
+
+
+ +
+
+

Authentication Information

+
+ + +
+
+
+
+ +
+
+
+ +
+
+

Important Note About Google Photos

+
+

All media items uploaded to Google Photos with rclone are stored in full resolution at original quality. These uploads will count towards storage in your Google Account.

+
+
+
+
+
+} \ No newline at end of file diff --git a/components/providers/source/gphotos.templ b/components/providers/source/gphotos.templ new file mode 100644 index 0000000..d1e0187 --- /dev/null +++ b/components/providers/source/gphotos.templ @@ -0,0 +1,220 @@ +package source + +templ GooglePhotosSourceForm() { +
+
+ +

+ Simple one-click authentication using rclone's shared credentials +

+
+ +
+
+ +
+
+ +
+ +
+

+ Client ID from Google Cloud Console +

+
+ +
+ +
+
+ +
+ +
+

+ Client Secret from Google Cloud Console +

+
+
+ +
+ +

+ Only request read-only access to your photos +

+
+ +
+ +
+
+ +
+ +
+

+ Only include photos uploaded after this year +

+
+ +
+ +

+ Include archived photos and videos in media listings +

+
+ +
+ +
+
+ +
+ +
+

+ Path within Google Photos to download files from +

+
+ +
+
+
+ +
+
+

Important: Authentication Required

+
+

After saving this configuration, you will need to authenticate with Google Photos.

+

The authentication process will require you to:

+
    +
  1. Visit a Google authorization URL
    2. +
  2. Sign in to your Google account
    3. +
  3. Grant permission to access your Google Photos
    4. +
  4. Copy the authorization code back to this application
    5. +
+

+ This is a one-time process for each configuration. The application will store your authorization token securely. +

+
+
+
+
+ +
+
+
+ +
+
+

Authentication Information

+
+ + +
+
+
+
+ +
+
+
+ +
+
+

Important Note About Google Photos

+
+

When downloading from Google Photos, be aware that some original metadata may not be preserved. Google Photos processes and may compress some images upon upload.

+
+
+
+
+
+} \ No newline at end of file diff --git a/internal/db/db.go b/internal/db/db.go index 527b4a6..09e295b 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -1,11 +1,11 @@ package db import ( - "encoding/json" "fmt" "os" "os/exec" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -74,6 +74,10 @@ type TransferConfig struct { SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form SourceDriveID string `form:"source_drive_id"` // For OneDrive SourceTeamDrive string `form:"source_team_drive"` // For Google Drive + // Google Photos source fields + SourceReadOnly *bool `form:"source_read_only"` // For Google Photos + SourceStartYear int `form:"source_start_year"` // For Google Photos + SourceIncludeArchived *bool `form:"source_include_archived"` // For Google Photos // General fields FilePattern string `gorm:"default:'*'" form:"file_pattern"` OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables @@ -100,8 +104,13 @@ type TransferConfig struct { DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form DestDriveID string `form:"dest_drive_id"` // For OneDrive DestTeamDrive string `form:"dest_team_drive"` // For Google Drive - // Google Drive authentication status - GoogleDriveAuthenticated *bool `gorm:"default:false"` + // Google Photos destination fields + DestReadOnly *bool `form:"dest_read_only"` // For Google Photos + DestStartYear int `form:"dest_start_year"` // For Google Photos + DestIncludeArchived *bool `form:"dest_include_archived"` // For Google Photos + // Security fields + UseBuiltinAuth *bool `form:"use_builtin_auth"` // For Google and other OAuth services + GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed // General fields ArchivePath string `form:"archive_path"` ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"` @@ -632,6 +641,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "team_drive", config.SourceTeamDrive) } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) + } + case "gphotos": + args := []string{ + "config", "create", sourceName, "google photos", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + // Only add client_id and client_secret if they're provided (not empty) + // This allows using rclone's built-in authentication + if config.SourceClientID != "" && config.SourceClientSecret != "" { + args = append(args, "client_id", config.SourceClientID) + args = append(args, "client_secret", config.SourceClientSecret) + } + + // Add read_only option if specified + if config.SourceReadOnly != nil && *config.SourceReadOnly { + args = append(args, "read_only", "true") + } + + // Add start_year if specified + if config.SourceStartYear > 0 { + args = append(args, "start_year", strconv.Itoa(config.SourceStartYear)) + } + + // Add include_archived if specified + if config.SourceIncludeArchived != nil && *config.SourceIncludeArchived { + args = append(args, "include_archived", "true") + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) @@ -829,6 +872,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { args = append(args, "root_folder_id", config.DestDriveID) } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) + } + case "gphotos": + args := []string{ + "config", "create", destName, "google photos", + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + + // Only add client_id and client_secret if they're provided (not empty) + // This allows using rclone's built-in authentication + if config.DestClientID != "" && config.DestClientSecret != "" { + args = append(args, "client_id", config.DestClientID) + args = append(args, "client_secret", config.DestClientSecret) + } + + // Add read_only option if specified + if config.DestReadOnly != nil && *config.DestReadOnly { + args = append(args, "read_only", "true") + } + + // Add start_year if specified + if config.DestStartYear > 0 { + args = append(args, "start_year", strconv.Itoa(config.DestStartYear)) + } + + // Add include_archived if specified + if config.DestIncludeArchived != nil && *config.DestIncludeArchived { + args = append(args, "include_archived", "true") + } + cmd := exec.Command(rclonePath, args...) if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output) @@ -1010,235 +1087,129 @@ func (db *DB) StoreGoogleDriveToken(configIDStr string, token string) error { // GenerateRcloneConfigWithToken generates a rclone config file for a transfer config with a provided token func (db *DB) GenerateRcloneConfigWithToken(config *TransferConfig, token string) error { - // Ensure we have a config directory - dataDir := os.Getenv("DATA_DIR") - if dataDir == "" { - dataDir = "./data" - } - configDir := filepath.Join(dataDir, "configs") - if err := os.MkdirAll(configDir, 0755); err != nil { - return err + // Get the config path + configPath := db.GetConfigRclonePath(config) + if configPath == "" { + return fmt.Errorf("failed to get config path") } - // Generate rclone config based on the config type - configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", config.ID)) + // Clean up the token to ensure it's a single line JSON + token = strings.TrimSpace(token) + token = strings.ReplaceAll(token, "\n", "") + token = strings.ReplaceAll(token, "\r", "") - // Create a new config content - var configContent strings.Builder + // Determine if this is a source or destination config + var configType, section, clientID, clientSecret string + var readOnly, includeArchived *bool + var startYear int - // First add the source configuration - sourceName := fmt.Sprintf("source_%d", config.ID) + if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" { + configType = config.DestinationType + section = "dest" + clientID = config.DestClientID + clientSecret = config.DestClientSecret + readOnly = config.DestReadOnly + startYear = config.DestStartYear + includeArchived = config.DestIncludeArchived + } else if config.SourceType == "gdrive" || config.SourceType == "gphotos" { + configType = config.SourceType + section = "source" + clientID = config.SourceClientID + clientSecret = config.SourceClientSecret + readOnly = config.SourceReadOnly + startYear = config.SourceStartYear + includeArchived = config.SourceIncludeArchived + } else { + return fmt.Errorf("config is not for Google Drive or Google Photos") + } - // Handle the source configuration based on type - switch config.SourceType { - case "google_drive": - // Create a Google Drive remote using the "source_ID" naming convention for sources - sourceSection := fmt.Sprintf("[%s]\ntype = drive\n", sourceName) + // Read the existing config + content, err := os.ReadFile(configPath) + if err != nil { + return fmt.Errorf("failed to read config file: %v", err) + } - // Add custom client ID and secret if provided - if config.SourceClientID != "" && config.SourceClientSecret != "" { - sourceSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n", - config.SourceClientID, config.SourceClientSecret) + // Prepare the section content + var sectionContent string + if configType == "gdrive" { + sectionContent = fmt.Sprintf("[%s_%d]\ntype = drive\n", section, config.ID) + if clientID != "" { + sectionContent += fmt.Sprintf("client_id = %s\n", clientID) } + if clientSecret != "" { + sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret) + } + sectionContent += fmt.Sprintf("token = %s\n", token) // Add team drive if specified - if config.SourceTeamDrive != "" { - sourceSection += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive) + if section == "source" && config.SourceTeamDrive != "" { + sectionContent += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive) + } else if section == "dest" && config.DestTeamDrive != "" { + sectionContent += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive) } - // Clean up token string to prevent syntax errors and ensure it's a single line JSON - // First, remove any whitespace from the beginning and end - cleanToken := strings.TrimSpace(token) + // Add read-only flag if specified + if readOnly != nil && *readOnly { + sectionContent += "read_only = true\n" + } + } else if configType == "gphotos" { + sectionContent = fmt.Sprintf("[%s_%d]\ntype = google photos\n", section, config.ID) + if clientID != "" { + sectionContent += fmt.Sprintf("client_id = %s\n", clientID) + } + if clientSecret != "" { + sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret) + } + sectionContent += fmt.Sprintf("token = %s\n", token) - // Check if it's already a JSON object - if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") { - // It's a JSON object, but we need to make sure it's a single line - var jsonObj map[string]interface{} - if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil { - // Successfully parsed the JSON, now re-marshal it as a compact single line - compactJSON, err := json.Marshal(jsonObj) - if err == nil { - // Use the compact JSON as the token - sourceSection += fmt.Sprintf("token = %s\n", string(compactJSON)) - } else { - // If there was an error re-marshaling, use the original but remove newlines - // Replace all newlines and carriage returns with empty string - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // If we couldn't parse the JSON, just remove newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // Not a valid JSON, try to fix it - // First ensure it starts and ends with braces - if !strings.HasPrefix(cleanToken, "{") { - cleanToken = "{" + cleanToken - } - if !strings.HasSuffix(cleanToken, "}") { - cleanToken = cleanToken + "}" - } - // Remove all newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - sourceSection += fmt.Sprintf("token = %s\n", singleLineToken) + // Add read-only flag if specified + if readOnly != nil && *readOnly { + sectionContent += "read_only = true\n" } - configContent.WriteString(sourceSection) - configContent.WriteString("\n") + // Add start year if specified + if startYear > 0 { + sectionContent += fmt.Sprintf("start_year = %d\n", startYear) + } - case "sftp", "s3", "minio", "b2", "smb", "ftp", "webdav", "nextcloud", "onedrive": - // For complex source types, use the GenerateRcloneConfig function - // to create a temporary file, read it, and then append that content - tempDir, err := os.MkdirTemp("", "gomft_temp") + // Add include_archived flag if specified and true + if includeArchived != nil && *includeArchived { + sectionContent += "include_archived = true\n" + } + } + + // Find the section in the existing config + sectionPattern := regexp.MustCompile(fmt.Sprintf(`\[%s_%d\][^\[]*`, section, config.ID)) + if sectionPattern.MatchString(string(content)) { + // Replace the existing section + newContent := sectionPattern.ReplaceAllString(string(content), sectionContent) + err = os.WriteFile(configPath, []byte(newContent), 0644) if err != nil { - return fmt.Errorf("failed to create temp directory: %v", err) + return fmt.Errorf("failed to write updated config file: %v", err) } - defer os.RemoveAll(tempDir) - - tempConfigPath := filepath.Join(tempDir, "temp_config.conf") - - // Create a temporary file with just the source configuration - tempContent := fmt.Sprintf("[%s]\ntype = local\n", sourceName) - if err := os.WriteFile(tempConfigPath, []byte(tempContent), 0600); err != nil { - return fmt.Errorf("failed to write temporary config: %v", err) - } - - // Get rclone path - rclonePath := os.Getenv("RCLONE_PATH") - if rclonePath == "" { - rclonePath = "rclone" - } - - // Use the appropriate rclone command to configure the source - var args []string - switch config.SourceType { - case "sftp": - args = []string{ - "config", "create", sourceName, "sftp", - "host", config.SourceHost, - "user", config.SourceUser, - "port", fmt.Sprintf("%d", config.SourcePort), - "--non-interactive", - "--config", tempConfigPath, - "--log-level", "ERROR", - } - if config.SourcePassword != "" { - args = append(args, "pass", config.SourcePassword) - } - if config.SourceKeyFile != "" { - args = append(args, "key_file", config.SourceKeyFile) - } - case "s3": - args = []string{ - "config", "create", sourceName, "s3", - "provider", "AWS", - "env_auth", "false", - "access_key_id", config.SourceAccessKey, - "secret_access_key", config.SourceSecretKey, - "region", config.SourceRegion, - "--non-interactive", - "--config", tempConfigPath, - "--log-level", "ERROR", - } - if config.SourceEndpoint != "" { - args = append(args, "endpoint", config.SourceEndpoint) - } - } - - // If we have arguments, execute the command - if len(args) > 0 { - cmd := exec.Command(rclonePath, args...) - if output, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output) - } - - // Read the generated config - sourceConfig, err := os.ReadFile(tempConfigPath) - if err != nil { - return fmt.Errorf("failed to read temporary config: %v", err) - } - - // Add it to our config content - configContent.WriteString(string(sourceConfig)) - configContent.WriteString("\n") - } - default: - // For local source or other simple types - sourceSection := fmt.Sprintf("[%s]\ntype = local\n\n", sourceName) - configContent.WriteString(sourceSection) - } - - // Now add the destination configuration - destName := fmt.Sprintf("dest_%d", config.ID) - - // Set up the destination section (only supporting Google Drive for now) - if config.DestinationType == "gdrive" || config.DestinationType == "google_drive" { - // Create a Google Drive remote using "dest_ID" naming convention - destSection := fmt.Sprintf("[%s]\ntype = drive\n", destName) - - // Add custom client ID and secret if provided - if config.DestClientID != "" && config.DestClientSecret != "" { - destSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n", - config.DestClientID, config.DestClientSecret) - } - - // Clean up token string to prevent syntax errors and ensure it's a single line JSON - // First, remove any whitespace from the beginning and end - cleanToken := strings.TrimSpace(token) - - // Check if it's already a JSON object - if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") { - // It's a JSON object, but we need to make sure it's a single line - var jsonObj map[string]interface{} - if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil { - // Successfully parsed the JSON, now re-marshal it as a compact single line - compactJSON, err := json.Marshal(jsonObj) - if err == nil { - // Use the compact JSON as the token - destSection += fmt.Sprintf("token = %s\n", string(compactJSON)) - } else { - // If there was an error re-marshaling, use the original but remove newlines - // Replace all newlines and carriage returns with empty string - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // If we couldn't parse the JSON, just remove newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - } else { - // Not a valid JSON, try to fix it - // First ensure it starts and ends with braces - if !strings.HasPrefix(cleanToken, "{") { - cleanToken = "{" + cleanToken - } - if !strings.HasSuffix(cleanToken, "}") { - cleanToken = cleanToken + "}" - } - // Remove all newlines and carriage returns - singleLineToken := strings.ReplaceAll(cleanToken, "\n", "") - singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "") - destSection += fmt.Sprintf("token = %s\n", singleLineToken) - } - - // Add to the config content - configContent.WriteString(destSection) } else { - // Add a simple local destination for testing or if no specific destination type is handled - destSection := fmt.Sprintf("[%s]\ntype = local\n", destName) - configContent.WriteString(destSection) + // Append the section to the config + file, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open config file for appending: %v", err) + } + defer file.Close() + + _, err = file.WriteString("\n" + sectionContent) + if err != nil { + return fmt.Errorf("failed to append to config file: %v", err) + } } - // Write the config file - return os.WriteFile(configPath, []byte(configContent.String()), 0644) + // Update the authentication status + authenticated := true + if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" { + config.SetGoogleAuthenticated(authenticated) + } else if config.SourceType == "gdrive" || config.SourceType == "gphotos" { + config.SetGoogleAuthenticated(authenticated) + } + + return nil } // GetIsAdmin returns the value of IsAdmin with a default if nil @@ -1306,19 +1277,26 @@ func (tc *TransferConfig) SetDestPassiveMode(value bool) { tc.DestPassiveMode = &value } -// GetGoogleDriveAuthenticated returns the value of GoogleDriveAuthenticated with a default if nil +// GetGoogleDriveAuthenticated returns whether the transfer config has been authenticated with Google Drive func (tc *TransferConfig) GetGoogleDriveAuthenticated() bool { - if tc.GoogleDriveAuthenticated == nil { - return false // Default to false if not set - } - return *tc.GoogleDriveAuthenticated + return tc.GoogleDriveAuthenticated != nil && *tc.GoogleDriveAuthenticated } -// SetGoogleDriveAuthenticated sets the GoogleDriveAuthenticated field +// SetGoogleDriveAuthenticated sets the Google Drive authentication status func (tc *TransferConfig) SetGoogleDriveAuthenticated(value bool) { tc.GoogleDriveAuthenticated = &value } +// GetGoogleAuthenticated is an alias for GetGoogleDriveAuthenticated for better semantics when working with Google Photos +func (tc *TransferConfig) GetGoogleAuthenticated() bool { + return tc.GetGoogleDriveAuthenticated() +} + +// SetGoogleAuthenticated is an alias for SetGoogleDriveAuthenticated for better semantics when working with Google Photos +func (tc *TransferConfig) SetGoogleAuthenticated(value bool) { + tc.SetGoogleDriveAuthenticated(value) +} + // GetArchiveEnabled returns the value of ArchiveEnabled with a default if nil func (tc *TransferConfig) GetArchiveEnabled() bool { if tc.ArchiveEnabled == nil { diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 0495838..c1e605c 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -1416,3 +1416,345 @@ func TestGoogleDriveTeamDrive(t *testing.T) { err = db.DeleteTransferConfig(teamDriveBothConfig.ID) assert.NoError(t, err) } + +func TestGooglePhotosTransferConfig(t *testing.T) { + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-test-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err := db.CreateUser(testUser) + if err != nil { + t.Fatalf("Failed to create user: %v", err) + } + + // Test 1: Create config with Google Photos as source + sourceReadOnly := true + sourceIncludeArchived := false + useBuiltinAuth := true + googleSourceConfig := &TransferConfig{ + Name: "Google Photos Source Test", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "google_client_id", + SourceClientSecret: "google_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &sourceIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/local/destination/path", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Set authenticated status + authenticated := true + googleSourceConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleSourceConfig) + assert.NoError(t, err) + assert.NotZero(t, googleSourceConfig.ID, "Config ID should be set after creation") + + // Test 2: Create config with Google Photos as destination + destReadOnly := false + destIncludeArchived := true + googleDestConfig := &TransferConfig{ + Name: "Google Photos Destination Test", + SourceType: "local", + SourcePath: "/local/source/path", + DestinationType: "gphotos", + DestinationPath: "/albums/upload", + DestClientID: "google_client_id", + DestClientSecret: "google_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2018, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.png", + CreatedBy: testUser.ID, + } + + // Set authenticated status + googleDestConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleDestConfig) + assert.NoError(t, err) + assert.NotZero(t, googleDestConfig.ID, "Config ID should be set after creation") + + // Test 3: Create config with Google Photos as both source and destination + googleBothConfig := &TransferConfig{ + Name: "Google Photos Both Test", + SourceType: "gphotos", + SourcePath: "/albums/source_album", + SourceClientID: "source_client_id", + SourceClientSecret: "source_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2020, + SourceIncludeArchived: &sourceIncludeArchived, + DestinationType: "gphotos", + DestinationPath: "/albums/dest_album", + DestClientID: "dest_client_id", + DestClientSecret: "dest_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2020, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.jpeg", + CreatedBy: testUser.ID, + } + + // Set authenticated status + googleBothConfig.GoogleDriveAuthenticated = &authenticated + + // Create the config + err = db.CreateTransferConfig(googleBothConfig) + assert.NoError(t, err) + assert.NotZero(t, googleBothConfig.ID, "Config ID should be set after creation") + + // Verify configs were created properly + retrievedConfig, err := db.GetTransferConfig(googleSourceConfig.ID) + assert.NoError(t, err) + assert.Equal(t, "gphotos", retrievedConfig.SourceType) + assert.Equal(t, sourceReadOnly, *retrievedConfig.SourceReadOnly) + assert.Equal(t, 2015, retrievedConfig.SourceStartYear) + assert.Equal(t, sourceIncludeArchived, *retrievedConfig.SourceIncludeArchived) + assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth) + assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated()) + + retrievedConfig, err = db.GetTransferConfig(googleDestConfig.ID) + assert.NoError(t, err) + assert.Equal(t, "gphotos", retrievedConfig.DestinationType) + assert.Equal(t, destReadOnly, *retrievedConfig.DestReadOnly) + assert.Equal(t, 2018, retrievedConfig.DestStartYear) + assert.Equal(t, destIncludeArchived, *retrievedConfig.DestIncludeArchived) + assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth) + assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated()) +} + +func TestGooglePhotosRcloneConfig(t *testing.T) { + // Create a temporary test directory + tempDir, err := os.MkdirTemp("", "gomft-test") + if err != nil { + t.Fatalf("Failed to create temp directory: %v", err) + } + defer os.RemoveAll(tempDir) + + // Set up data directory + dataDir := filepath.Join(tempDir, "data") + configDir := filepath.Join(dataDir, "configs") + err = os.MkdirAll(configDir, 0755) + if err != nil { + t.Fatalf("Failed to create config directory: %v", err) + } + + // Set DATA_DIR environment variable + oldDataDir := os.Getenv("DATA_DIR") + defer os.Setenv("DATA_DIR", oldDataDir) + os.Setenv("DATA_DIR", dataDir) + + // Initialize test database + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-rclone-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err = db.CreateUser(testUser) + assert.NoError(t, err) + + // TEST 1: Google Photos as source with standard options + sourceReadOnly := true + sourceIncludeArchived := false + useBuiltinAuth := true + gphotosSourceConfig := &TransferConfig{ + ID: 1, // Force ID for predictable config path + Name: "Google Photos Source Config", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "test_client_id", + SourceClientSecret: "test_client_secret", + SourceReadOnly: &sourceReadOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &sourceIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/tmp/destination", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Generate rclone config + err = db.GenerateRcloneConfig(gphotosSourceConfig) + assert.NoError(t, err) + + // Check if config file exists + configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosSourceConfig.ID)) + _, err = os.Stat(configPath) + assert.NoError(t, err, "Config file should exist") + + // Read config file content + content, err := os.ReadFile(configPath) + assert.NoError(t, err) + configContent := string(content) + + // Check for Google Photos source section + assert.Contains(t, configContent, "[source_1]") + assert.Contains(t, configContent, "type = google photos") + assert.Contains(t, configContent, "client_id = test_client_id") + assert.Contains(t, configContent, "client_secret = test_client_secret") + assert.Contains(t, configContent, "read_only = true") + assert.Contains(t, configContent, "start_year = 2015") + assert.NotContains(t, configContent, "include_archived = true") // This should be false and not included + + // TEST 2: Google Photos as destination with authenticated token + destReadOnly := false + destIncludeArchived := true + gphotosDestConfig := &TransferConfig{ + ID: 2, // Force ID for predictable config path + Name: "Google Photos Destination Config", + SourceType: "local", + SourcePath: "/tmp/source", + DestinationType: "gphotos", + DestinationPath: "/albums/upload", + DestClientID: "dest_client_id", + DestClientSecret: "dest_client_secret", + DestReadOnly: &destReadOnly, + DestStartYear: 2018, + DestIncludeArchived: &destIncludeArchived, + UseBuiltinAuth: &useBuiltinAuth, + FilePattern: "*.png", + CreatedBy: testUser.ID, + } + + // Set authentication status + authenticated := true + gphotosDestConfig.GoogleDriveAuthenticated = &authenticated + + // Generate config first (needed for token update) + err = db.GenerateRcloneConfig(gphotosDestConfig) + assert.NoError(t, err) + + // Now test token handling with GenerateRcloneConfigWithToken + testToken := `{"access_token":"test-token","token_type":"Bearer","refresh_token":"test-refresh","expiry":"2023-12-31T23:59:59Z"}` + err = db.GenerateRcloneConfigWithToken(gphotosDestConfig, testToken) + assert.NoError(t, err) + + // Check updated config + configPath = filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosDestConfig.ID)) + _, err = os.Stat(configPath) + assert.NoError(t, err, "Config file should exist") + + // Read config file content + content, err = os.ReadFile(configPath) + assert.NoError(t, err) + configContent = string(content) + + // Check for Google Photos destination section with token + assert.Contains(t, configContent, "type = google photos") + assert.Contains(t, configContent, "client_id = dest_client_id") + assert.Contains(t, configContent, "client_secret = dest_client_secret") + assert.Contains(t, configContent, "token = {") + assert.Contains(t, configContent, "access_token") + assert.Contains(t, configContent, "test-token") + assert.Contains(t, configContent, "refresh_token") + assert.Contains(t, configContent, "test-refresh") + assert.Contains(t, configContent, "include_archived = true") + assert.NotContains(t, configContent, "read_only = false") // This should be false and not included +} + +func TestGooglePhotosAuthentication(t *testing.T) { + // Initialize test database + db := setupTestDB(t) + + // Create a test user + testUser := &User{ + Email: fmt.Sprintf("gphotos-auth-%d@example.com", time.Now().UnixNano()), + PasswordHash: "hashed_password", + LastPasswordChange: time.Now(), + } + err := db.CreateUser(testUser) + assert.NoError(t, err) + + // Create a transfer config with Google Photos + readOnly := true + includeArchived := false + useBuiltinAuth := true + gPhotosConfig := &TransferConfig{ + Name: "Test Google Photos Auth", + SourceType: "gphotos", + SourcePath: "/albums/vacation", + SourceClientID: "test_client_id", + SourceClientSecret: "test_client_secret", + SourceReadOnly: &readOnly, + SourceStartYear: 2015, + SourceIncludeArchived: &includeArchived, + UseBuiltinAuth: &useBuiltinAuth, + DestinationType: "local", + DestinationPath: "/tmp/destination", + FilePattern: "*.jpg", + CreatedBy: testUser.ID, + } + + // Create the config + err = db.CreateTransferConfig(gPhotosConfig) + assert.NoError(t, err) + assert.NotZero(t, gPhotosConfig.ID) + + // Test initial authentication state + // Should be false when first created + authenticated := gPhotosConfig.GetGoogleAuthenticated() + assert.False(t, authenticated) + t.Logf("Initial GoogleDriveAuthenticated value: %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Test generic Google authentication method (new) + gPhotosConfig.SetGoogleAuthenticated(true) + t.Logf("After SetGoogleAuthenticated(true): %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(gPhotosConfig) + assert.NoError(t, err) + t.Logf("After UpdateTransferConfig: %v", gPhotosConfig.GoogleDriveAuthenticated) + + // Verify authentication status is updated + updatedConfig, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config GoogleDriveAuthenticated: %v", updatedConfig.GoogleDriveAuthenticated) + assert.True(t, updatedConfig.GetGoogleAuthenticated()) + + // Verify it can be unset + updatedConfig.SetGoogleAuthenticated(false) + t.Logf("After SetGoogleAuthenticated(false): %v", updatedConfig.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(updatedConfig) + assert.NoError(t, err) + + // Verify authentication status is updated + updatedConfig2, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config2 GoogleDriveAuthenticated: %v", updatedConfig2.GoogleDriveAuthenticated) + assert.False(t, updatedConfig2.GetGoogleAuthenticated()) + + // Test with the old method naming for backward compatibility + updatedConfig2.SetGoogleDriveAuthenticated(true) + t.Logf("After SetGoogleDriveAuthenticated(true): %v", updatedConfig2.GoogleDriveAuthenticated) + + // Save the updated config to the database + err = db.UpdateTransferConfig(updatedConfig2) + assert.NoError(t, err) + + // Verify authentication status is updated when using the old method + updatedConfig3, err := db.GetTransferConfig(gPhotosConfig.ID) + assert.NoError(t, err) + t.Logf("Retrieved config3 GoogleDriveAuthenticated: %v", updatedConfig3.GoogleDriveAuthenticated) + assert.True(t, updatedConfig3.GetGoogleAuthenticated()) + assert.True(t, updatedConfig3.GetGoogleDriveAuthenticated()) +} diff --git a/internal/db/migrations/add_google_photos_support.go b/internal/db/migrations/add_google_photos_support.go new file mode 100644 index 0000000..2e2255d --- /dev/null +++ b/internal/db/migrations/add_google_photos_support.go @@ -0,0 +1,61 @@ +package migrations + +import ( + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +// AddGooglePhotosSupport adds Google Photos related fields to the transfer_configs table +func AddGooglePhotosSupport() *gormigrate.Migration { + return &gormigrate.Migration{ + ID: "20240518_add_google_photos_support", + Migrate: func(tx *gorm.DB) error { + // Add Google Photos source fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_read_only BOOLEAN DEFAULT false").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_start_year INTEGER DEFAULT 0").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_include_archived BOOLEAN DEFAULT false").Error; err != nil { + return err + } + + // Add Google Photos destination fields + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_read_only BOOLEAN DEFAULT false").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_start_year INTEGER DEFAULT 0").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_include_archived BOOLEAN DEFAULT false").Error; err != nil { + return err + } + + // Add OAuth field + return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN use_builtin_auth BOOLEAN DEFAULT true").Error + }, + Rollback: func(tx *gorm.DB) error { + // Remove all added columns in reverse order + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN use_builtin_auth").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_include_archived").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_start_year").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_read_only").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_include_archived").Error; err != nil { + return err + } + if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_start_year").Error; err != nil { + return err + } + return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_read_only").Error + }, + } +} diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index b0953f4..fea061e 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -17,6 +17,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate { UpdateSkipProcessedFilesToNullable(), AddWebhookSupport(), AddGoogleDriveAuthenticated(), + AddGooglePhotosSupport(), } return gormigrate.New(db, gormigrate.DefaultOptions, migrations) diff --git a/internal/scheduler/scheduler.go b/internal/scheduler/scheduler.go index 6f4b615..61da064 100644 --- a/internal/scheduler/scheduler.go +++ b/internal/scheduler/scheduler.go @@ -535,6 +535,12 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig, if maxConcurrent < 1 { maxConcurrent = 1 // Default to 1 if not set } + + // Limit Google Photos to 1 concurrent transfers + if config.SourceType == "gphotos" || config.DestinationType == "gphotos" { + maxConcurrent = 1 + } + s.log.LogInfo("Using %d concurrent transfers for job %d, config %d", maxConcurrent, job.ID, config.ID) // Create wait group for concurrent processing diff --git a/internal/web/handlers/config_handlers.go b/internal/web/handlers/config_handlers.go index 42241f1..dc886a5 100644 --- a/internal/web/handlers/config_handlers.go +++ b/internal/web/handlers/config_handlers.go @@ -101,6 +101,27 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) { destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true" config.DestPassiveMode = &destPassiveModeValue + // Google Photos specific fields + destReadOnlyVal := c.Request.FormValue("dest_read_only") + destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true" + config.DestReadOnly = &destReadOnlyValue + + sourceReadOnlyVal := c.Request.FormValue("source_read_only") + sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true" + config.SourceReadOnly = &sourceReadOnlyValue + + destIncludeArchivedVal := c.Request.FormValue("dest_include_archived") + destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true" + config.DestIncludeArchived = &destIncludeArchivedValue + + sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived") + sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" + config.SourceIncludeArchived = &sourceIncludeArchivedValue + + useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth") + useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true" + config.UseBuiltinAuth = &useBuiltinAuthValue + if err := h.DB.Create(&config).Error; err != nil { log.Printf("Error creating config: %v", err) c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to create config: %v", err)) @@ -171,6 +192,27 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) { destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true" config.DestPassiveMode = &destPassiveModeValue + // Google Photos specific fields + destReadOnlyVal := c.Request.FormValue("dest_read_only") + destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true" + config.DestReadOnly = &destReadOnlyValue + + sourceReadOnlyVal := c.Request.FormValue("source_read_only") + sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true" + config.SourceReadOnly = &sourceReadOnlyValue + + destIncludeArchivedVal := c.Request.FormValue("dest_include_archived") + destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true" + config.DestIncludeArchived = &destIncludeArchivedValue + + sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived") + sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true" + config.SourceIncludeArchived = &sourceIncludeArchivedValue + + useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth") + useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true" + config.UseBuiltinAuth = &useBuiltinAuthValue + // Preserve fields that shouldn't be updated config.CreatedBy = oldConfig.CreatedBy diff --git a/internal/web/handlers/gdrive_handlers.go b/internal/web/handlers/gdrive_handlers.go index 0f4ced4..96ed2c1 100644 --- a/internal/web/handlers/gdrive_handlers.go +++ b/internal/web/handlers/gdrive_handlers.go @@ -36,9 +36,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { return } - // Ensure it's a Google Drive configuration - if config.DestinationType != "gdrive" { - RenderErrorPage(c, "Not a Google Drive configuration", "The selected configuration is not set up for Google Drive") + // Ensure it's a Google Drive or Google Photos configuration + if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" { + RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos") return } @@ -81,9 +81,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { // Define the redirect URI for our callback redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL) - // Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV - clientID := os.Getenv("GDRIVE_CLIENT_ID") - clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET") + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") if clientID == "" || clientSecret == "" { // Check if we have client credentials in the existing config file @@ -109,7 +109,7 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { clientSecret = existingClientSecret } else { // If we still can't find a matching secret, show an error - RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.") + RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.") return } } @@ -123,13 +123,28 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) { // Store config ID in cookie for use during callback c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true) + // Determine the appropriate scope based on destination type + var scope string + if config.DestinationType == "gphotos" { + // Read-only access is handled elsewhere in the config; here we need the full auth scope + scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary") + } else { + // Default to Google Drive scope + scope = url.QueryEscape("https://www.googleapis.com/auth/drive") + } + // Create a config file with redirect URI-based auth - configContent := fmt.Sprintf(`[temp_gdrive] -type = drive + configType := "drive" + if config.DestinationType == "gphotos" { + configType = "google photos" + } + + configContent := fmt.Sprintf(`[temp_%s] +type = %s client_id = %s client_secret = %s redirect_url = %s -`, clientID, clientSecret, redirectURI) +`, config.DestinationType, configType, clientID, clientSecret, redirectURI) // Write the config file if err := os.WriteFile(tempConfigPath, []byte(configContent), 0644); err != nil { @@ -138,7 +153,6 @@ redirect_url = %s } // Direct Google OAuth URL with our redirect - scope := url.QueryEscape("https://www.googleapis.com/auth/drive") authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s", url.QueryEscape(clientID), url.QueryEscape(redirectURI), @@ -205,9 +219,9 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { return } - // Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV - clientID := os.Getenv("GDRIVE_CLIENT_ID") - clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET") + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") if clientID == "" || clientSecret == "" { // Check if we have client credentials in the existing config file @@ -233,7 +247,7 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { clientSecret = existingClientSecret } else { // If we still can't find a matching secret, show an error - RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.") + RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.") return } } @@ -313,7 +327,13 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) { c.SetCookie("gdrive_config_id", "", -1, "/", "", false, true) // Redirect to the config list with a success message - c.Redirect(http.StatusFound, "/configs?status=gdrive_auth_success") + var successParam string + if config.DestinationType == "gphotos" { + successParam = "gphotos_auth_success" + } else { + successParam = "gdrive_auth_success" + } + c.Redirect(http.StatusFound, fmt.Sprintf("/configs?status=%s", successParam)) } // HandleGDriveTokenProcess processes a Google Drive token directly from a URL parameter diff --git a/internal/web/handlers/gdrive_handlers_test.go b/internal/web/handlers/gdrive_handlers_test.go new file mode 100644 index 0000000..30d507f --- /dev/null +++ b/internal/web/handlers/gdrive_handlers_test.go @@ -0,0 +1,429 @@ +package handlers + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/starfleetcptn/gomft/internal/db" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// DBInterface defines the methods we need to mock for our tests +type DBInterface interface { + GetTransferConfig(id uint) (*db.TransferConfig, error) + GetConfigRclonePath(config *db.TransferConfig) string + GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error + GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string) +} + +// MockDB is a mock implementation of the DB interface for testing +type MockDB struct { + mock.Mock +} + +// Implement the necessary methods from the DB interface for our tests +func (m *MockDB) GetTransferConfig(id uint) (*db.TransferConfig, error) { + args := m.Called(id) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*db.TransferConfig), args.Error(1) +} + +func (m *MockDB) GetConfigRclonePath(config *db.TransferConfig) string { + args := m.Called(config) + return args.String(0) +} + +func (m *MockDB) GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error { + args := m.Called(config, token) + return args.Error(0) +} + +func (m *MockDB) GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string) { + args := m.Called(config) + return args.String(0), args.String(1) +} + +// MockHandlers is a modified version of Handlers that accepts our mock DB +type MockHandlers struct { + DB DBInterface +} + +// HandleGDriveAuth is a copy of the original method but using our interface +func (h *MockHandlers) HandleGDriveAuth(c *gin.Context) { + // Get the config ID from the query parameter + configIDStr := c.Param("id") + if configIDStr == "" { + RenderErrorPage(c, "Missing configuration ID", "") + return + } + + configID, err := strconv.ParseUint(configIDStr, 10, 64) + if err != nil { + RenderErrorPage(c, "Invalid configuration ID", err.Error()) + return + } + + // Get the configuration + config, err := h.DB.GetTransferConfig(uint(configID)) + if err != nil { + RenderErrorPage(c, "Configuration not found", err.Error()) + return + } + + // Ensure it's a Google Drive or Google Photos configuration + if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" { + RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos") + return + } + + // Prepare for OAuth + dataDir := os.Getenv("DATA_DIR") + if dataDir == "" { + dataDir = "./data" + } + + // Get Rclone Config Path + rcloneConfigPath := h.DB.GetConfigRclonePath(config) + if rcloneConfigPath == "" { + RenderErrorPage(c, "Rclone config not found", "The selected configuration does not have a valid rclone config") + return + } + + // Create a temporary config file for authentication + tempConfigDir := filepath.Join(dataDir, "temp") + if err := os.MkdirAll(tempConfigDir, 0755); err != nil { + RenderErrorPage(c, "Failed to create temporary directory", err.Error()) + return + } + + tempConfigPath := filepath.Join(tempConfigDir, fmt.Sprintf("gdrive_auth_%d.conf", config.ID)) + + // Store the temporary config path in a cookie + c.SetCookie("gdrive_temp_config", tempConfigPath, 3600, "/", "", false, true) + + // Get base URL for redirect URI + baseURL := os.Getenv("BASE_URL") + if baseURL == "" { + // Try to detect the base URL from the request + scheme := "http" + if c.Request.TLS != nil { + scheme = "https" + } + baseURL = fmt.Sprintf("%s://%s", scheme, c.Request.Host) + } + + // Define the redirect URI for our callback + redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL) + + // Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV + clientID := os.Getenv("GOOGLE_CLIENT_ID") + clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET") + + if clientID == "" || clientSecret == "" { + // Check if we have client credentials in the existing config file + existingClientID, existingClientSecret := h.DB.GetGDriveCredentialsFromConfig(config) + + if existingClientID != "" && existingClientSecret != "" { + // Use credentials from existing config + clientID = existingClientID + clientSecret = existingClientSecret + } else { + // fallback to rclone client ID and secret + clientID = "202264815644.apps.googleusercontent.com" + clientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ" + } + } + + // Generate state parameter for security (to prevent CSRF) + state := fmt.Sprintf("gomft_%d_%d", config.ID, time.Now().Unix()) + + // Store state in cookie for validation during callback + c.SetCookie("gdrive_auth_state", state, 3600, "/", "", false, true) + + // Store config ID in cookie for use during callback + c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true) + + // Determine the appropriate scope based on destination type + var scope string + if config.DestinationType == "gphotos" { + // Read-only access is handled elsewhere in the config; here we need the full auth scope + scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary") + } else { + // Default to Google Drive scope + scope = url.QueryEscape("https://www.googleapis.com/auth/drive") + } + + // Direct Google OAuth URL with our redirect + authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s", + url.QueryEscape(clientID), + url.QueryEscape(redirectURI), + scope, + url.QueryEscape(state)) + + // Redirect the user to Google's auth page directly + c.Redirect(http.StatusFound, authURL) +} + +// HandleGDriveAuthCallback handles the callback from Google OAuth +func (h *MockHandlers) HandleGDriveAuthCallback(c *gin.Context) { + // Get auth code from query parameters + authCode := c.Query("code") + if authCode == "" { + RenderErrorPage(c, "Authentication failed", "No authorization code received from Google") + return + } + + // Verify state parameter to prevent CSRF + state := c.Query("state") + storedState, err := c.Cookie("gdrive_auth_state") + if err != nil || state != storedState { + RenderErrorPage(c, "Authentication failed", "Invalid state parameter") + return + } + + // Get config ID from cookie + configIDStr, err := c.Cookie("gdrive_config_id") + if err != nil { + RenderErrorPage(c, "Authentication failed", "Unable to retrieve configuration ID") + return + } + + configID, err := strconv.ParseUint(configIDStr, 10, 64) + if err != nil { + RenderErrorPage(c, "Invalid configuration ID", err.Error()) + return + } + + // Get the configuration + config, err := h.DB.GetTransferConfig(uint(configID)) + if err != nil { + RenderErrorPage(c, "Failed to get configuration", err.Error()) + return + } + + // For testing purposes, we'll simulate a successful token exchange + // In a real implementation, we would exchange the auth code for a token + mockToken := `{"access_token":"test_access_token","refresh_token":"test_refresh_token","expiry":"2023-12-31T23:59:59Z"}` + + // Update the config with the token + err = h.DB.GenerateRcloneConfigWithToken(config, mockToken) + if err != nil { + RenderErrorPage(c, "Failed to update configuration", err.Error()) + return + } + + // Redirect to the config edit page + c.Redirect(http.StatusFound, fmt.Sprintf("/configs/edit/%d", config.ID)) +} + +func setupTestRouter() (*gin.Engine, *MockDB) { + gin.SetMode(gin.TestMode) + router := gin.New() + mockDB := new(MockDB) + handlers := &MockHandlers{ + DB: mockDB, + } + + router.GET("/configs/gdrive/:id", handlers.HandleGDriveAuth) + router.GET("/configs/gdrive-callback", handlers.HandleGDriveAuthCallback) + + return router, mockDB +} + +func TestHandleGDriveAuth_GoogleDrive(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 1, + DestinationType: "gdrive", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil) + mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf") + mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret") + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/1", nil) + router.ServeHTTP(w, req) + + // Assertions + assert.Equal(t, http.StatusFound, w.Code) + + // Verify the redirect URL + location := w.Header().Get("Location") + assert.Contains(t, location, "accounts.google.com/o/oauth2/auth") + assert.Contains(t, location, "drive") + assert.Contains(t, location, "test_client_id") + + // Verify cookies were set + cookies := w.Result().Cookies() + assert.GreaterOrEqual(t, len(cookies), 3) + + // Check if state cookie exists + stateFound := false + for _, cookie := range cookies { + if cookie.Name == "gdrive_auth_state" { + stateFound = true + break + } + } + assert.True(t, stateFound) +} + +func TestHandleGDriveAuth_GooglePhotos(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 2, + DestinationType: "gphotos", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(2)).Return(testConfig, nil) + mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf") + mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret") + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/2", nil) + router.ServeHTTP(w, req) + + // Assertions + assert.Equal(t, http.StatusFound, w.Code) + + // Verify the redirect URL + location := w.Header().Get("Location") + assert.Contains(t, location, "accounts.google.com/o/oauth2/auth") + assert.Contains(t, location, "photoslibrary") + assert.Contains(t, location, "test_client_id") + + // Verify cookies were set + cookies := w.Result().Cookies() + assert.GreaterOrEqual(t, len(cookies), 3) + + // Check if state cookie exists + stateFound := false + for _, cookie := range cookies { + if cookie.Name == "gdrive_auth_state" { + stateFound = true + break + } + } + assert.True(t, stateFound) +} + +func TestHandleGDriveAuthCallback(t *testing.T) { + // Setup test environment + router, mockDB := setupTestRouter() + + // Create a temporary directory for testing + tempDir, err := os.MkdirTemp("", "gdrive-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tempDir) + + // Create a temporary config file + tempConfigPath := filepath.Join(tempDir, "temp_config.conf") + if err := os.WriteFile(tempConfigPath, []byte("test config"), 0644); err != nil { + t.Fatal(err) + } + + // Test state and config ID + testState := "gomft_1_12345" + testConfigID := "1" + + // Create a test config + testConfig := &db.TransferConfig{ + ID: 1, + DestinationType: "gphotos", + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil) + mockDB.On("GenerateRcloneConfigWithToken", testConfig, mock.Anything).Return(nil) + + // Create test request with auth code and state + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive-callback?code=test_auth_code&state="+testState, nil) + + // Add required cookies to the request + req.AddCookie(&http.Cookie{Name: "gdrive_auth_state", Value: testState}) + req.AddCookie(&http.Cookie{Name: "gdrive_config_id", Value: testConfigID}) + req.AddCookie(&http.Cookie{Name: "gdrive_temp_config", Value: tempConfigPath}) + + // Send the request + router.ServeHTTP(w, req) + + // We expect a redirect on successful auth + assert.Equal(t, http.StatusFound, w.Code) + + // Should redirect to the config edit page + location := w.Header().Get("Location") + assert.Contains(t, location, "/configs/edit/1") +} + +func TestHandleGDriveAuth_InvalidConfig(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Set up mock expectations for a non-existent config + mockDB.On("GetTransferConfig", uint(999)).Return(nil, fmt.Errorf("config not found")) + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/999", nil) + router.ServeHTTP(w, req) + + // Assertions - should render error page + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "Configuration not found") +} + +func TestHandleGDriveAuth_NonGoogleConfig(t *testing.T) { + // Setup + router, mockDB := setupTestRouter() + + // Create a non-Google test config + testConfig := &db.TransferConfig{ + ID: 3, + DestinationType: "s3", // Not Google Drive or Photos + } + + // Set up mock expectations + mockDB.On("GetTransferConfig", uint(3)).Return(testConfig, nil) + + // Create test request + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/configs/gdrive/3", nil) + router.ServeHTTP(w, req) + + // Assertions - should render error page + assert.Equal(t, http.StatusOK, w.Code) + assert.Contains(t, w.Body.String(), "Not a Google configuration") +} + +// RenderErrorPage renders an error page with the given message +func RenderErrorPage(c *gin.Context, title string, details string) { + // Here we'd typically use a component for error display + // For now, we'll just render a simple HTML error page for testing + errorHTML := fmt.Sprintf("

Error: %s

%s

", title, details) + c.Data(http.StatusOK, "text/html", []byte(errorHTML)) +}