mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-09-20 13:30:51 +02:00
feat: Add Google Photos support for transfer configurations
- Introduce Google Photos as a source and destination option in the configuration forms. - Implement authentication handling and configuration management for Google Photos. - Update database schema and migrations to include new fields related to Google Photos. - Enhance UI components to support Google Photos-specific settings and forms. - Add tests for Google Photos integration, ensuring proper functionality and authentication flow. - Limit concurrent transfers for Google Photos to ensure compliance with API restrictions.
This commit is contained in:
StarFleetCPTN
+196
-218
@@ -1,11 +1,11 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -74,6 +74,10 @@ type TransferConfig struct {
|
||||
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
||||
SourceDriveID string `form:"source_drive_id"` // For OneDrive
|
||||
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
|
||||
// Google Photos source fields
|
||||
SourceReadOnly *bool `form:"source_read_only"` // For Google Photos
|
||||
SourceStartYear int `form:"source_start_year"` // For Google Photos
|
||||
SourceIncludeArchived *bool `form:"source_include_archived"` // For Google Photos
|
||||
// General fields
|
||||
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
|
||||
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
|
||||
@@ -100,8 +104,13 @@ type TransferConfig struct {
|
||||
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
|
||||
DestDriveID string `form:"dest_drive_id"` // For OneDrive
|
||||
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
|
||||
// Google Drive authentication status
|
||||
GoogleDriveAuthenticated *bool `gorm:"default:false"`
|
||||
// Google Photos destination fields
|
||||
DestReadOnly *bool `form:"dest_read_only"` // For Google Photos
|
||||
DestStartYear int `form:"dest_start_year"` // For Google Photos
|
||||
DestIncludeArchived *bool `form:"dest_include_archived"` // For Google Photos
|
||||
// Security fields
|
||||
UseBuiltinAuth *bool `form:"use_builtin_auth"` // For Google and other OAuth services
|
||||
GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed
|
||||
// General fields
|
||||
ArchivePath string `form:"archive_path"`
|
||||
ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"`
|
||||
@@ -632,6 +641,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
|
||||
args = append(args, "team_drive", config.SourceTeamDrive)
|
||||
}
|
||||
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
|
||||
}
|
||||
case "gphotos":
|
||||
args := []string{
|
||||
"config", "create", sourceName, "google photos",
|
||||
"--non-interactive",
|
||||
"--config", configPath,
|
||||
"--log-level", "ERROR",
|
||||
}
|
||||
|
||||
// Only add client_id and client_secret if they're provided (not empty)
|
||||
// This allows using rclone's built-in authentication
|
||||
if config.SourceClientID != "" && config.SourceClientSecret != "" {
|
||||
args = append(args, "client_id", config.SourceClientID)
|
||||
args = append(args, "client_secret", config.SourceClientSecret)
|
||||
}
|
||||
|
||||
// Add read_only option if specified
|
||||
if config.SourceReadOnly != nil && *config.SourceReadOnly {
|
||||
args = append(args, "read_only", "true")
|
||||
}
|
||||
|
||||
// Add start_year if specified
|
||||
if config.SourceStartYear > 0 {
|
||||
args = append(args, "start_year", strconv.Itoa(config.SourceStartYear))
|
||||
}
|
||||
|
||||
// Add include_archived if specified
|
||||
if config.SourceIncludeArchived != nil && *config.SourceIncludeArchived {
|
||||
args = append(args, "include_archived", "true")
|
||||
}
|
||||
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
|
||||
@@ -829,6 +872,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
|
||||
args = append(args, "root_folder_id", config.DestDriveID)
|
||||
}
|
||||
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
|
||||
}
|
||||
case "gphotos":
|
||||
args := []string{
|
||||
"config", "create", destName, "google photos",
|
||||
"--non-interactive",
|
||||
"--config", configPath,
|
||||
"--log-level", "ERROR",
|
||||
}
|
||||
|
||||
// Only add client_id and client_secret if they're provided (not empty)
|
||||
// This allows using rclone's built-in authentication
|
||||
if config.DestClientID != "" && config.DestClientSecret != "" {
|
||||
args = append(args, "client_id", config.DestClientID)
|
||||
args = append(args, "client_secret", config.DestClientSecret)
|
||||
}
|
||||
|
||||
// Add read_only option if specified
|
||||
if config.DestReadOnly != nil && *config.DestReadOnly {
|
||||
args = append(args, "read_only", "true")
|
||||
}
|
||||
|
||||
// Add start_year if specified
|
||||
if config.DestStartYear > 0 {
|
||||
args = append(args, "start_year", strconv.Itoa(config.DestStartYear))
|
||||
}
|
||||
|
||||
// Add include_archived if specified
|
||||
if config.DestIncludeArchived != nil && *config.DestIncludeArchived {
|
||||
args = append(args, "include_archived", "true")
|
||||
}
|
||||
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
|
||||
@@ -1010,235 +1087,129 @@ func (db *DB) StoreGoogleDriveToken(configIDStr string, token string) error {
|
||||
|
||||
// GenerateRcloneConfigWithToken generates a rclone config file for a transfer config with a provided token
|
||||
func (db *DB) GenerateRcloneConfigWithToken(config *TransferConfig, token string) error {
|
||||
// Ensure we have a config directory
|
||||
dataDir := os.Getenv("DATA_DIR")
|
||||
if dataDir == "" {
|
||||
dataDir = "./data"
|
||||
}
|
||||
configDir := filepath.Join(dataDir, "configs")
|
||||
if err := os.MkdirAll(configDir, 0755); err != nil {
|
||||
return err
|
||||
// Get the config path
|
||||
configPath := db.GetConfigRclonePath(config)
|
||||
if configPath == "" {
|
||||
return fmt.Errorf("failed to get config path")
|
||||
}
|
||||
|
||||
// Generate rclone config based on the config type
|
||||
configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", config.ID))
|
||||
// Clean up the token to ensure it's a single line JSON
|
||||
token = strings.TrimSpace(token)
|
||||
token = strings.ReplaceAll(token, "\n", "")
|
||||
token = strings.ReplaceAll(token, "\r", "")
|
||||
|
||||
// Create a new config content
|
||||
var configContent strings.Builder
|
||||
// Determine if this is a source or destination config
|
||||
var configType, section, clientID, clientSecret string
|
||||
var readOnly, includeArchived *bool
|
||||
var startYear int
|
||||
|
||||
// First add the source configuration
|
||||
sourceName := fmt.Sprintf("source_%d", config.ID)
|
||||
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
|
||||
configType = config.DestinationType
|
||||
section = "dest"
|
||||
clientID = config.DestClientID
|
||||
clientSecret = config.DestClientSecret
|
||||
readOnly = config.DestReadOnly
|
||||
startYear = config.DestStartYear
|
||||
includeArchived = config.DestIncludeArchived
|
||||
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
|
||||
configType = config.SourceType
|
||||
section = "source"
|
||||
clientID = config.SourceClientID
|
||||
clientSecret = config.SourceClientSecret
|
||||
readOnly = config.SourceReadOnly
|
||||
startYear = config.SourceStartYear
|
||||
includeArchived = config.SourceIncludeArchived
|
||||
} else {
|
||||
return fmt.Errorf("config is not for Google Drive or Google Photos")
|
||||
}
|
||||
|
||||
// Handle the source configuration based on type
|
||||
switch config.SourceType {
|
||||
case "google_drive":
|
||||
// Create a Google Drive remote using the "source_ID" naming convention for sources
|
||||
sourceSection := fmt.Sprintf("[%s]\ntype = drive\n", sourceName)
|
||||
// Read the existing config
|
||||
content, err := os.ReadFile(configPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read config file: %v", err)
|
||||
}
|
||||
|
||||
// Add custom client ID and secret if provided
|
||||
if config.SourceClientID != "" && config.SourceClientSecret != "" {
|
||||
sourceSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n",
|
||||
config.SourceClientID, config.SourceClientSecret)
|
||||
// Prepare the section content
|
||||
var sectionContent string
|
||||
if configType == "gdrive" {
|
||||
sectionContent = fmt.Sprintf("[%s_%d]\ntype = drive\n", section, config.ID)
|
||||
if clientID != "" {
|
||||
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
|
||||
}
|
||||
if clientSecret != "" {
|
||||
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
|
||||
}
|
||||
sectionContent += fmt.Sprintf("token = %s\n", token)
|
||||
|
||||
// Add team drive if specified
|
||||
if config.SourceTeamDrive != "" {
|
||||
sourceSection += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive)
|
||||
if section == "source" && config.SourceTeamDrive != "" {
|
||||
sectionContent += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive)
|
||||
} else if section == "dest" && config.DestTeamDrive != "" {
|
||||
sectionContent += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive)
|
||||
}
|
||||
|
||||
// Clean up token string to prevent syntax errors and ensure it's a single line JSON
|
||||
// First, remove any whitespace from the beginning and end
|
||||
cleanToken := strings.TrimSpace(token)
|
||||
// Add read-only flag if specified
|
||||
if readOnly != nil && *readOnly {
|
||||
sectionContent += "read_only = true\n"
|
||||
}
|
||||
} else if configType == "gphotos" {
|
||||
sectionContent = fmt.Sprintf("[%s_%d]\ntype = google photos\n", section, config.ID)
|
||||
if clientID != "" {
|
||||
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
|
||||
}
|
||||
if clientSecret != "" {
|
||||
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
|
||||
}
|
||||
sectionContent += fmt.Sprintf("token = %s\n", token)
|
||||
|
||||
// Check if it's already a JSON object
|
||||
if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") {
|
||||
// It's a JSON object, but we need to make sure it's a single line
|
||||
var jsonObj map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil {
|
||||
// Successfully parsed the JSON, now re-marshal it as a compact single line
|
||||
compactJSON, err := json.Marshal(jsonObj)
|
||||
if err == nil {
|
||||
// Use the compact JSON as the token
|
||||
sourceSection += fmt.Sprintf("token = %s\n", string(compactJSON))
|
||||
} else {
|
||||
// If there was an error re-marshaling, use the original but remove newlines
|
||||
// Replace all newlines and carriage returns with empty string
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
}
|
||||
} else {
|
||||
// If we couldn't parse the JSON, just remove newlines and carriage returns
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
}
|
||||
} else {
|
||||
// Not a valid JSON, try to fix it
|
||||
// First ensure it starts and ends with braces
|
||||
if !strings.HasPrefix(cleanToken, "{") {
|
||||
cleanToken = "{" + cleanToken
|
||||
}
|
||||
if !strings.HasSuffix(cleanToken, "}") {
|
||||
cleanToken = cleanToken + "}"
|
||||
}
|
||||
// Remove all newlines and carriage returns
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
// Add read-only flag if specified
|
||||
if readOnly != nil && *readOnly {
|
||||
sectionContent += "read_only = true\n"
|
||||
}
|
||||
|
||||
configContent.WriteString(sourceSection)
|
||||
configContent.WriteString("\n")
|
||||
// Add start year if specified
|
||||
if startYear > 0 {
|
||||
sectionContent += fmt.Sprintf("start_year = %d\n", startYear)
|
||||
}
|
||||
|
||||
case "sftp", "s3", "minio", "b2", "smb", "ftp", "webdav", "nextcloud", "onedrive":
|
||||
// For complex source types, use the GenerateRcloneConfig function
|
||||
// to create a temporary file, read it, and then append that content
|
||||
tempDir, err := os.MkdirTemp("", "gomft_temp")
|
||||
// Add include_archived flag if specified and true
|
||||
if includeArchived != nil && *includeArchived {
|
||||
sectionContent += "include_archived = true\n"
|
||||
}
|
||||
}
|
||||
|
||||
// Find the section in the existing config
|
||||
sectionPattern := regexp.MustCompile(fmt.Sprintf(`\[%s_%d\][^\[]*`, section, config.ID))
|
||||
if sectionPattern.MatchString(string(content)) {
|
||||
// Replace the existing section
|
||||
newContent := sectionPattern.ReplaceAllString(string(content), sectionContent)
|
||||
err = os.WriteFile(configPath, []byte(newContent), 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp directory: %v", err)
|
||||
return fmt.Errorf("failed to write updated config file: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
tempConfigPath := filepath.Join(tempDir, "temp_config.conf")
|
||||
|
||||
// Create a temporary file with just the source configuration
|
||||
tempContent := fmt.Sprintf("[%s]\ntype = local\n", sourceName)
|
||||
if err := os.WriteFile(tempConfigPath, []byte(tempContent), 0600); err != nil {
|
||||
return fmt.Errorf("failed to write temporary config: %v", err)
|
||||
}
|
||||
|
||||
// Get rclone path
|
||||
rclonePath := os.Getenv("RCLONE_PATH")
|
||||
if rclonePath == "" {
|
||||
rclonePath = "rclone"
|
||||
}
|
||||
|
||||
// Use the appropriate rclone command to configure the source
|
||||
var args []string
|
||||
switch config.SourceType {
|
||||
case "sftp":
|
||||
args = []string{
|
||||
"config", "create", sourceName, "sftp",
|
||||
"host", config.SourceHost,
|
||||
"user", config.SourceUser,
|
||||
"port", fmt.Sprintf("%d", config.SourcePort),
|
||||
"--non-interactive",
|
||||
"--config", tempConfigPath,
|
||||
"--log-level", "ERROR",
|
||||
}
|
||||
if config.SourcePassword != "" {
|
||||
args = append(args, "pass", config.SourcePassword)
|
||||
}
|
||||
if config.SourceKeyFile != "" {
|
||||
args = append(args, "key_file", config.SourceKeyFile)
|
||||
}
|
||||
case "s3":
|
||||
args = []string{
|
||||
"config", "create", sourceName, "s3",
|
||||
"provider", "AWS",
|
||||
"env_auth", "false",
|
||||
"access_key_id", config.SourceAccessKey,
|
||||
"secret_access_key", config.SourceSecretKey,
|
||||
"region", config.SourceRegion,
|
||||
"--non-interactive",
|
||||
"--config", tempConfigPath,
|
||||
"--log-level", "ERROR",
|
||||
}
|
||||
if config.SourceEndpoint != "" {
|
||||
args = append(args, "endpoint", config.SourceEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// If we have arguments, execute the command
|
||||
if len(args) > 0 {
|
||||
cmd := exec.Command(rclonePath, args...)
|
||||
if output, err := cmd.CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
|
||||
}
|
||||
|
||||
// Read the generated config
|
||||
sourceConfig, err := os.ReadFile(tempConfigPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read temporary config: %v", err)
|
||||
}
|
||||
|
||||
// Add it to our config content
|
||||
configContent.WriteString(string(sourceConfig))
|
||||
configContent.WriteString("\n")
|
||||
}
|
||||
default:
|
||||
// For local source or other simple types
|
||||
sourceSection := fmt.Sprintf("[%s]\ntype = local\n\n", sourceName)
|
||||
configContent.WriteString(sourceSection)
|
||||
}
|
||||
|
||||
// Now add the destination configuration
|
||||
destName := fmt.Sprintf("dest_%d", config.ID)
|
||||
|
||||
// Set up the destination section (only supporting Google Drive for now)
|
||||
if config.DestinationType == "gdrive" || config.DestinationType == "google_drive" {
|
||||
// Create a Google Drive remote using "dest_ID" naming convention
|
||||
destSection := fmt.Sprintf("[%s]\ntype = drive\n", destName)
|
||||
|
||||
// Add custom client ID and secret if provided
|
||||
if config.DestClientID != "" && config.DestClientSecret != "" {
|
||||
destSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n",
|
||||
config.DestClientID, config.DestClientSecret)
|
||||
}
|
||||
|
||||
// Clean up token string to prevent syntax errors and ensure it's a single line JSON
|
||||
// First, remove any whitespace from the beginning and end
|
||||
cleanToken := strings.TrimSpace(token)
|
||||
|
||||
// Check if it's already a JSON object
|
||||
if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") {
|
||||
// It's a JSON object, but we need to make sure it's a single line
|
||||
var jsonObj map[string]interface{}
|
||||
if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil {
|
||||
// Successfully parsed the JSON, now re-marshal it as a compact single line
|
||||
compactJSON, err := json.Marshal(jsonObj)
|
||||
if err == nil {
|
||||
// Use the compact JSON as the token
|
||||
destSection += fmt.Sprintf("token = %s\n", string(compactJSON))
|
||||
} else {
|
||||
// If there was an error re-marshaling, use the original but remove newlines
|
||||
// Replace all newlines and carriage returns with empty string
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
}
|
||||
} else {
|
||||
// If we couldn't parse the JSON, just remove newlines and carriage returns
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
}
|
||||
} else {
|
||||
// Not a valid JSON, try to fix it
|
||||
// First ensure it starts and ends with braces
|
||||
if !strings.HasPrefix(cleanToken, "{") {
|
||||
cleanToken = "{" + cleanToken
|
||||
}
|
||||
if !strings.HasSuffix(cleanToken, "}") {
|
||||
cleanToken = cleanToken + "}"
|
||||
}
|
||||
// Remove all newlines and carriage returns
|
||||
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
|
||||
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
|
||||
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
|
||||
}
|
||||
|
||||
// Add to the config content
|
||||
configContent.WriteString(destSection)
|
||||
} else {
|
||||
// Add a simple local destination for testing or if no specific destination type is handled
|
||||
destSection := fmt.Sprintf("[%s]\ntype = local\n", destName)
|
||||
configContent.WriteString(destSection)
|
||||
// Append the section to the config
|
||||
file, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open config file for appending: %v", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = file.WriteString("\n" + sectionContent)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to append to config file: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Write the config file
|
||||
return os.WriteFile(configPath, []byte(configContent.String()), 0644)
|
||||
// Update the authentication status
|
||||
authenticated := true
|
||||
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
|
||||
config.SetGoogleAuthenticated(authenticated)
|
||||
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
|
||||
config.SetGoogleAuthenticated(authenticated)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetIsAdmin returns the value of IsAdmin with a default if nil
|
||||
@@ -1306,19 +1277,26 @@ func (tc *TransferConfig) SetDestPassiveMode(value bool) {
|
||||
tc.DestPassiveMode = &value
|
||||
}
|
||||
|
||||
// GetGoogleDriveAuthenticated returns the value of GoogleDriveAuthenticated with a default if nil
|
||||
// GetGoogleDriveAuthenticated returns whether the transfer config has been authenticated with Google Drive
|
||||
func (tc *TransferConfig) GetGoogleDriveAuthenticated() bool {
|
||||
if tc.GoogleDriveAuthenticated == nil {
|
||||
return false // Default to false if not set
|
||||
}
|
||||
return *tc.GoogleDriveAuthenticated
|
||||
return tc.GoogleDriveAuthenticated != nil && *tc.GoogleDriveAuthenticated
|
||||
}
|
||||
|
||||
// SetGoogleDriveAuthenticated sets the GoogleDriveAuthenticated field
|
||||
// SetGoogleDriveAuthenticated sets the Google Drive authentication status
|
||||
func (tc *TransferConfig) SetGoogleDriveAuthenticated(value bool) {
|
||||
tc.GoogleDriveAuthenticated = &value
|
||||
}
|
||||
|
||||
// GetGoogleAuthenticated is an alias for GetGoogleDriveAuthenticated for better semantics when working with Google Photos
|
||||
func (tc *TransferConfig) GetGoogleAuthenticated() bool {
|
||||
return tc.GetGoogleDriveAuthenticated()
|
||||
}
|
||||
|
||||
// SetGoogleAuthenticated is an alias for SetGoogleDriveAuthenticated for better semantics when working with Google Photos
|
||||
func (tc *TransferConfig) SetGoogleAuthenticated(value bool) {
|
||||
tc.SetGoogleDriveAuthenticated(value)
|
||||
}
|
||||
|
||||
// GetArchiveEnabled returns the value of ArchiveEnabled with a default if nil
|
||||
func (tc *TransferConfig) GetArchiveEnabled() bool {
|
||||
if tc.ArchiveEnabled == nil {
|
||||
|
||||
@@ -1416,3 +1416,345 @@ func TestGoogleDriveTeamDrive(t *testing.T) {
|
||||
err = db.DeleteTransferConfig(teamDriveBothConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestGooglePhotosTransferConfig(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
|
||||
// Create a test user
|
||||
testUser := &User{
|
||||
Email: fmt.Sprintf("gphotos-test-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hashed_password",
|
||||
LastPasswordChange: time.Now(),
|
||||
}
|
||||
err := db.CreateUser(testUser)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create user: %v", err)
|
||||
}
|
||||
|
||||
// Test 1: Create config with Google Photos as source
|
||||
sourceReadOnly := true
|
||||
sourceIncludeArchived := false
|
||||
useBuiltinAuth := true
|
||||
googleSourceConfig := &TransferConfig{
|
||||
Name: "Google Photos Source Test",
|
||||
SourceType: "gphotos",
|
||||
SourcePath: "/albums/vacation",
|
||||
SourceClientID: "google_client_id",
|
||||
SourceClientSecret: "google_client_secret",
|
||||
SourceReadOnly: &sourceReadOnly,
|
||||
SourceStartYear: 2015,
|
||||
SourceIncludeArchived: &sourceIncludeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
DestinationType: "local",
|
||||
DestinationPath: "/local/destination/path",
|
||||
FilePattern: "*.jpg",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Set authenticated status
|
||||
authenticated := true
|
||||
googleSourceConfig.GoogleDriveAuthenticated = &authenticated
|
||||
|
||||
// Create the config
|
||||
err = db.CreateTransferConfig(googleSourceConfig)
|
||||
assert.NoError(t, err)
|
||||
assert.NotZero(t, googleSourceConfig.ID, "Config ID should be set after creation")
|
||||
|
||||
// Test 2: Create config with Google Photos as destination
|
||||
destReadOnly := false
|
||||
destIncludeArchived := true
|
||||
googleDestConfig := &TransferConfig{
|
||||
Name: "Google Photos Destination Test",
|
||||
SourceType: "local",
|
||||
SourcePath: "/local/source/path",
|
||||
DestinationType: "gphotos",
|
||||
DestinationPath: "/albums/upload",
|
||||
DestClientID: "google_client_id",
|
||||
DestClientSecret: "google_client_secret",
|
||||
DestReadOnly: &destReadOnly,
|
||||
DestStartYear: 2018,
|
||||
DestIncludeArchived: &destIncludeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
FilePattern: "*.png",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Set authenticated status
|
||||
googleDestConfig.GoogleDriveAuthenticated = &authenticated
|
||||
|
||||
// Create the config
|
||||
err = db.CreateTransferConfig(googleDestConfig)
|
||||
assert.NoError(t, err)
|
||||
assert.NotZero(t, googleDestConfig.ID, "Config ID should be set after creation")
|
||||
|
||||
// Test 3: Create config with Google Photos as both source and destination
|
||||
googleBothConfig := &TransferConfig{
|
||||
Name: "Google Photos Both Test",
|
||||
SourceType: "gphotos",
|
||||
SourcePath: "/albums/source_album",
|
||||
SourceClientID: "source_client_id",
|
||||
SourceClientSecret: "source_client_secret",
|
||||
SourceReadOnly: &sourceReadOnly,
|
||||
SourceStartYear: 2020,
|
||||
SourceIncludeArchived: &sourceIncludeArchived,
|
||||
DestinationType: "gphotos",
|
||||
DestinationPath: "/albums/dest_album",
|
||||
DestClientID: "dest_client_id",
|
||||
DestClientSecret: "dest_client_secret",
|
||||
DestReadOnly: &destReadOnly,
|
||||
DestStartYear: 2020,
|
||||
DestIncludeArchived: &destIncludeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
FilePattern: "*.jpeg",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Set authenticated status
|
||||
googleBothConfig.GoogleDriveAuthenticated = &authenticated
|
||||
|
||||
// Create the config
|
||||
err = db.CreateTransferConfig(googleBothConfig)
|
||||
assert.NoError(t, err)
|
||||
assert.NotZero(t, googleBothConfig.ID, "Config ID should be set after creation")
|
||||
|
||||
// Verify configs were created properly
|
||||
retrievedConfig, err := db.GetTransferConfig(googleSourceConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "gphotos", retrievedConfig.SourceType)
|
||||
assert.Equal(t, sourceReadOnly, *retrievedConfig.SourceReadOnly)
|
||||
assert.Equal(t, 2015, retrievedConfig.SourceStartYear)
|
||||
assert.Equal(t, sourceIncludeArchived, *retrievedConfig.SourceIncludeArchived)
|
||||
assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth)
|
||||
assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated())
|
||||
|
||||
retrievedConfig, err = db.GetTransferConfig(googleDestConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "gphotos", retrievedConfig.DestinationType)
|
||||
assert.Equal(t, destReadOnly, *retrievedConfig.DestReadOnly)
|
||||
assert.Equal(t, 2018, retrievedConfig.DestStartYear)
|
||||
assert.Equal(t, destIncludeArchived, *retrievedConfig.DestIncludeArchived)
|
||||
assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth)
|
||||
assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated())
|
||||
}
|
||||
|
||||
func TestGooglePhotosRcloneConfig(t *testing.T) {
|
||||
// Create a temporary test directory
|
||||
tempDir, err := os.MkdirTemp("", "gomft-test")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create temp directory: %v", err)
|
||||
}
|
||||
defer os.RemoveAll(tempDir)
|
||||
|
||||
// Set up data directory
|
||||
dataDir := filepath.Join(tempDir, "data")
|
||||
configDir := filepath.Join(dataDir, "configs")
|
||||
err = os.MkdirAll(configDir, 0755)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create config directory: %v", err)
|
||||
}
|
||||
|
||||
// Set DATA_DIR environment variable
|
||||
oldDataDir := os.Getenv("DATA_DIR")
|
||||
defer os.Setenv("DATA_DIR", oldDataDir)
|
||||
os.Setenv("DATA_DIR", dataDir)
|
||||
|
||||
// Initialize test database
|
||||
db := setupTestDB(t)
|
||||
|
||||
// Create a test user
|
||||
testUser := &User{
|
||||
Email: fmt.Sprintf("gphotos-rclone-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hashed_password",
|
||||
LastPasswordChange: time.Now(),
|
||||
}
|
||||
err = db.CreateUser(testUser)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// TEST 1: Google Photos as source with standard options
|
||||
sourceReadOnly := true
|
||||
sourceIncludeArchived := false
|
||||
useBuiltinAuth := true
|
||||
gphotosSourceConfig := &TransferConfig{
|
||||
ID: 1, // Force ID for predictable config path
|
||||
Name: "Google Photos Source Config",
|
||||
SourceType: "gphotos",
|
||||
SourcePath: "/albums/vacation",
|
||||
SourceClientID: "test_client_id",
|
||||
SourceClientSecret: "test_client_secret",
|
||||
SourceReadOnly: &sourceReadOnly,
|
||||
SourceStartYear: 2015,
|
||||
SourceIncludeArchived: &sourceIncludeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
DestinationType: "local",
|
||||
DestinationPath: "/tmp/destination",
|
||||
FilePattern: "*.jpg",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Generate rclone config
|
||||
err = db.GenerateRcloneConfig(gphotosSourceConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check if config file exists
|
||||
configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosSourceConfig.ID))
|
||||
_, err = os.Stat(configPath)
|
||||
assert.NoError(t, err, "Config file should exist")
|
||||
|
||||
// Read config file content
|
||||
content, err := os.ReadFile(configPath)
|
||||
assert.NoError(t, err)
|
||||
configContent := string(content)
|
||||
|
||||
// Check for Google Photos source section
|
||||
assert.Contains(t, configContent, "[source_1]")
|
||||
assert.Contains(t, configContent, "type = google photos")
|
||||
assert.Contains(t, configContent, "client_id = test_client_id")
|
||||
assert.Contains(t, configContent, "client_secret = test_client_secret")
|
||||
assert.Contains(t, configContent, "read_only = true")
|
||||
assert.Contains(t, configContent, "start_year = 2015")
|
||||
assert.NotContains(t, configContent, "include_archived = true") // This should be false and not included
|
||||
|
||||
// TEST 2: Google Photos as destination with authenticated token
|
||||
destReadOnly := false
|
||||
destIncludeArchived := true
|
||||
gphotosDestConfig := &TransferConfig{
|
||||
ID: 2, // Force ID for predictable config path
|
||||
Name: "Google Photos Destination Config",
|
||||
SourceType: "local",
|
||||
SourcePath: "/tmp/source",
|
||||
DestinationType: "gphotos",
|
||||
DestinationPath: "/albums/upload",
|
||||
DestClientID: "dest_client_id",
|
||||
DestClientSecret: "dest_client_secret",
|
||||
DestReadOnly: &destReadOnly,
|
||||
DestStartYear: 2018,
|
||||
DestIncludeArchived: &destIncludeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
FilePattern: "*.png",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Set authentication status
|
||||
authenticated := true
|
||||
gphotosDestConfig.GoogleDriveAuthenticated = &authenticated
|
||||
|
||||
// Generate config first (needed for token update)
|
||||
err = db.GenerateRcloneConfig(gphotosDestConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Now test token handling with GenerateRcloneConfigWithToken
|
||||
testToken := `{"access_token":"test-token","token_type":"Bearer","refresh_token":"test-refresh","expiry":"2023-12-31T23:59:59Z"}`
|
||||
err = db.GenerateRcloneConfigWithToken(gphotosDestConfig, testToken)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Check updated config
|
||||
configPath = filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosDestConfig.ID))
|
||||
_, err = os.Stat(configPath)
|
||||
assert.NoError(t, err, "Config file should exist")
|
||||
|
||||
// Read config file content
|
||||
content, err = os.ReadFile(configPath)
|
||||
assert.NoError(t, err)
|
||||
configContent = string(content)
|
||||
|
||||
// Check for Google Photos destination section with token
|
||||
assert.Contains(t, configContent, "type = google photos")
|
||||
assert.Contains(t, configContent, "client_id = dest_client_id")
|
||||
assert.Contains(t, configContent, "client_secret = dest_client_secret")
|
||||
assert.Contains(t, configContent, "token = {")
|
||||
assert.Contains(t, configContent, "access_token")
|
||||
assert.Contains(t, configContent, "test-token")
|
||||
assert.Contains(t, configContent, "refresh_token")
|
||||
assert.Contains(t, configContent, "test-refresh")
|
||||
assert.Contains(t, configContent, "include_archived = true")
|
||||
assert.NotContains(t, configContent, "read_only = false") // This should be false and not included
|
||||
}
|
||||
|
||||
func TestGooglePhotosAuthentication(t *testing.T) {
|
||||
// Initialize test database
|
||||
db := setupTestDB(t)
|
||||
|
||||
// Create a test user
|
||||
testUser := &User{
|
||||
Email: fmt.Sprintf("gphotos-auth-%d@example.com", time.Now().UnixNano()),
|
||||
PasswordHash: "hashed_password",
|
||||
LastPasswordChange: time.Now(),
|
||||
}
|
||||
err := db.CreateUser(testUser)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Create a transfer config with Google Photos
|
||||
readOnly := true
|
||||
includeArchived := false
|
||||
useBuiltinAuth := true
|
||||
gPhotosConfig := &TransferConfig{
|
||||
Name: "Test Google Photos Auth",
|
||||
SourceType: "gphotos",
|
||||
SourcePath: "/albums/vacation",
|
||||
SourceClientID: "test_client_id",
|
||||
SourceClientSecret: "test_client_secret",
|
||||
SourceReadOnly: &readOnly,
|
||||
SourceStartYear: 2015,
|
||||
SourceIncludeArchived: &includeArchived,
|
||||
UseBuiltinAuth: &useBuiltinAuth,
|
||||
DestinationType: "local",
|
||||
DestinationPath: "/tmp/destination",
|
||||
FilePattern: "*.jpg",
|
||||
CreatedBy: testUser.ID,
|
||||
}
|
||||
|
||||
// Create the config
|
||||
err = db.CreateTransferConfig(gPhotosConfig)
|
||||
assert.NoError(t, err)
|
||||
assert.NotZero(t, gPhotosConfig.ID)
|
||||
|
||||
// Test initial authentication state
|
||||
// Should be false when first created
|
||||
authenticated := gPhotosConfig.GetGoogleAuthenticated()
|
||||
assert.False(t, authenticated)
|
||||
t.Logf("Initial GoogleDriveAuthenticated value: %v", gPhotosConfig.GoogleDriveAuthenticated)
|
||||
|
||||
// Test generic Google authentication method (new)
|
||||
gPhotosConfig.SetGoogleAuthenticated(true)
|
||||
t.Logf("After SetGoogleAuthenticated(true): %v", gPhotosConfig.GoogleDriveAuthenticated)
|
||||
|
||||
// Save the updated config to the database
|
||||
err = db.UpdateTransferConfig(gPhotosConfig)
|
||||
assert.NoError(t, err)
|
||||
t.Logf("After UpdateTransferConfig: %v", gPhotosConfig.GoogleDriveAuthenticated)
|
||||
|
||||
// Verify authentication status is updated
|
||||
updatedConfig, err := db.GetTransferConfig(gPhotosConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
t.Logf("Retrieved config GoogleDriveAuthenticated: %v", updatedConfig.GoogleDriveAuthenticated)
|
||||
assert.True(t, updatedConfig.GetGoogleAuthenticated())
|
||||
|
||||
// Verify it can be unset
|
||||
updatedConfig.SetGoogleAuthenticated(false)
|
||||
t.Logf("After SetGoogleAuthenticated(false): %v", updatedConfig.GoogleDriveAuthenticated)
|
||||
|
||||
// Save the updated config to the database
|
||||
err = db.UpdateTransferConfig(updatedConfig)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify authentication status is updated
|
||||
updatedConfig2, err := db.GetTransferConfig(gPhotosConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
t.Logf("Retrieved config2 GoogleDriveAuthenticated: %v", updatedConfig2.GoogleDriveAuthenticated)
|
||||
assert.False(t, updatedConfig2.GetGoogleAuthenticated())
|
||||
|
||||
// Test with the old method naming for backward compatibility
|
||||
updatedConfig2.SetGoogleDriveAuthenticated(true)
|
||||
t.Logf("After SetGoogleDriveAuthenticated(true): %v", updatedConfig2.GoogleDriveAuthenticated)
|
||||
|
||||
// Save the updated config to the database
|
||||
err = db.UpdateTransferConfig(updatedConfig2)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Verify authentication status is updated when using the old method
|
||||
updatedConfig3, err := db.GetTransferConfig(gPhotosConfig.ID)
|
||||
assert.NoError(t, err)
|
||||
t.Logf("Retrieved config3 GoogleDriveAuthenticated: %v", updatedConfig3.GoogleDriveAuthenticated)
|
||||
assert.True(t, updatedConfig3.GetGoogleAuthenticated())
|
||||
assert.True(t, updatedConfig3.GetGoogleDriveAuthenticated())
|
||||
}
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"github.com/go-gormigrate/gormigrate/v2"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
// AddGooglePhotosSupport adds Google Photos related fields to the transfer_configs table
|
||||
func AddGooglePhotosSupport() *gormigrate.Migration {
|
||||
return &gormigrate.Migration{
|
||||
ID: "20240518_add_google_photos_support",
|
||||
Migrate: func(tx *gorm.DB) error {
|
||||
// Add Google Photos source fields
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_read_only BOOLEAN DEFAULT false").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_start_year INTEGER DEFAULT 0").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_include_archived BOOLEAN DEFAULT false").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add Google Photos destination fields
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_read_only BOOLEAN DEFAULT false").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_start_year INTEGER DEFAULT 0").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_include_archived BOOLEAN DEFAULT false").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Add OAuth field
|
||||
return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN use_builtin_auth BOOLEAN DEFAULT true").Error
|
||||
},
|
||||
Rollback: func(tx *gorm.DB) error {
|
||||
// Remove all added columns in reverse order
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN use_builtin_auth").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_include_archived").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_start_year").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_read_only").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_include_archived").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_start_year").Error; err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_read_only").Error
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
|
||||
UpdateSkipProcessedFilesToNullable(),
|
||||
AddWebhookSupport(),
|
||||
AddGoogleDriveAuthenticated(),
|
||||
AddGooglePhotosSupport(),
|
||||
}
|
||||
|
||||
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
|
||||
|
||||
Reference in New Issue
Block a user