feat: Add Google Photos support for transfer configurations

- Introduce Google Photos as a source and destination option in the configuration forms.
- Implement authentication handling and configuration management for Google Photos.
- Update database schema and migrations to include new fields related to Google Photos.
- Enhance UI components to support Google Photos-specific settings and forms.
- Add tests for Google Photos integration, ensuring proper functionality and authentication flow.
- Limit concurrent transfers for Google Photos to ensure compliance with API restrictions.
This commit is contained in:
StarFleetCPTN
2025-03-15 23:16:49 -07:00
parent 8607f2098a
commit d68a80bd78
13 changed files with 1604 additions and 238 deletions
+46 -1
View File
@@ -7,6 +7,7 @@ import (
"github.com/starfleetcptn/gomft/components/providers/source"
"github.com/starfleetcptn/gomft/components/providers/destination"
"github.com/starfleetcptn/gomft/components/providers/common"
"strconv"
)
type ConfigFormData struct {
@@ -44,6 +45,10 @@ func getInitialData(config *db.TransferConfig) string {
sourceClientSecret := ""
sourceDriveId := ""
sourceTeamDrive := ""
// Google Photos source fields
sourceReadOnly := false
sourceStartYear, _ := strconv.Atoi(getCurrentYear())
sourceIncludeArchived := false
filePattern := ""
outputPattern := "${filename}"
@@ -68,6 +73,10 @@ func getInitialData(config *db.TransferConfig) string {
destClientSecret := ""
destDriveId := ""
destTeamDrive := ""
// Google Photos destination fields
destReadOnly := false
destStartYear, _ := strconv.Atoi(getCurrentYear()) // Default to current year int
destIncludeArchived := false
archivePath := ""
archiveEnabled := false
@@ -103,6 +112,15 @@ func getInitialData(config *db.TransferConfig) string {
sourceDriveId = config.SourceDriveID
sourceTeamDrive = config.SourceTeamDrive
// Google Photos source fields
if config.SourceReadOnly != nil {
sourceReadOnly = *config.SourceReadOnly
}
sourceStartYear = config.SourceStartYear
if config.SourceIncludeArchived != nil {
sourceIncludeArchived = *config.SourceIncludeArchived
}
filePattern = config.FilePattern
outputPattern = config.OutputPattern
@@ -129,13 +147,24 @@ func getInitialData(config *db.TransferConfig) string {
destDriveId = config.DestDriveID
destTeamDrive = config.DestTeamDrive
// Google Photos destination fields
if config.DestReadOnly != nil {
destReadOnly = *config.DestReadOnly
}
destStartYear = config.DestStartYear
if config.DestIncludeArchived != nil {
destIncludeArchived = *config.DestIncludeArchived
}
archivePath = config.ArchivePath
archiveEnabled = config.GetArchiveEnabled()
deleteAfterTransfer = config.GetDeleteAfterTransfer()
skipProcessedFiles = config.GetSkipProcessedFiles()
maxConcurrentTransfers = config.MaxConcurrentTransfers
rcloneFlags = config.RcloneFlags
if destClientId != "" || destClientSecret != "" {
if config.UseBuiltinAuth != nil {
useBuiltinAuth = *config.UseBuiltinAuth
} else if destClientId != "" || destClientSecret != "" {
useBuiltinAuth = false
}
}
@@ -163,6 +192,9 @@ func getInitialData(config *db.TransferConfig) string {
sourceClientSecret: '%s',
sourceDriveId: '%s',
sourceTeamDrive: '%s',
sourceReadOnly: %v,
sourceStartYear: %d,
sourceIncludeArchived: %v,
filePattern: '%s',
outputPattern: '%s',
@@ -187,6 +219,10 @@ func getInitialData(config *db.TransferConfig) string {
destClientSecret: '%s',
destDriveId: '%s',
destTeamDrive: '%s',
destReadOnly: %v,
destStartYear: %d,
destIncludeArchived: %v,
useBuiltinAuth: %v,
archivePath: '%s',
@@ -199,10 +235,12 @@ func getInitialData(config *db.TransferConfig) string {
name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
sourceBucket, sourceRegion, sourceAccessKey, sourceSecretKey, sourceEndpoint, sourceShare, sourceDomain, sourcePassiveMode,
sourceClientId, sourceClientSecret, sourceDriveId, sourceTeamDrive,
sourceReadOnly, sourceStartYear, sourceIncludeArchived,
filePattern, outputPattern,
destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
destClientId, destClientSecret, destDriveId, destTeamDrive,
destReadOnly, destStartYear, destIncludeArchived,
useBuiltinAuth,
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
}
@@ -287,6 +325,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) {
@source.GoogleDriveSourceForm()
</template>
<template x-if="sourceType === 'gphotos'">
@source.GooglePhotosSourceForm()
</template>
<!-- File pattern fields -->
@common.FilePatternFields()
@@ -330,6 +372,9 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) {
@destination.GoogleDriveDestinationForm()
</template>
<template x-if="destinationType === 'gphotos'">
@destination.GooglePhotosDestinationForm()
</template>
<!-- Archive options -->
@common.ArchiveOptions()
+3 -3
View File
@@ -270,7 +270,7 @@ templ Configs(ctx context.Context, data ConfigsData) {
</p>
<!-- Google Drive Authentication Badge -->
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && !config.GetGoogleDriveAuthenticated() {
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && !config.GetGoogleAuthenticated() {
<span class="ml-2 px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-yellow-100 text-yellow-800 dark:bg-yellow-800 dark:text-yellow-100">
<i class="fas fa-exclamation-triangle mr-1 flex items-center"></i>
Authentication Required
@@ -278,7 +278,7 @@ templ Configs(ctx context.Context, data ConfigsData) {
}
<!-- Google Drive Authentication Status Indicator -->
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.DestinationType == "drive") && config.GetGoogleDriveAuthenticated() {
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && config.GetGoogleAuthenticated() {
<span class="ml-2 px-2 inline-flex text-xs leading-5 font-semibold rounded-full bg-green-100 text-green-800 dark:bg-green-800 dark:text-green-100">
<i class="fas fa-check-circle mr-1 flex items-center"></i>
Authenticated
@@ -287,7 +287,7 @@ templ Configs(ctx context.Context, data ConfigsData) {
</div>
<div class="ml-2 flex-shrink-0 flex space-x-2">
<!-- Google Drive Authentication Button -->
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive") && !config.GetGoogleDriveAuthenticated() {
if (config.DestinationType == "gdrive" || config.SourceType == "gdrive" || config.SourceType == "gphotos" || config.DestinationType == "gphotos") && !config.GetGoogleAuthenticated() {
<a href={ templ.SafeURL(fmt.Sprintf("/configs/%d/gdrive-auth", config.ID)) } class="btn-warning btn-sm">
<i class="fab fa-google-drive mr-1"></i>
Authenticate
+2
View File
@@ -195,6 +195,7 @@ templ SourceSelection() {
<option value="nextcloud">NextCloud</option>
<option value="webdav">WebDAV</option>
<option value="google_drive">Google Drive</option>
<option value="gphotos">Google Photos</option>
</select>
</div>
</div>
@@ -219,6 +220,7 @@ templ DestinationSelection() {
<option value="nextcloud">NextCloud</option>
<option value="webdav">WebDAV</option>
<option value="gdrive">Google Drive</option>
<option value="gphotos">Google Photos</option>
</select>
</div>
</div>
@@ -0,0 +1,220 @@
package destination
templ GooglePhotosDestinationForm() {
<div class="space-y-6" x-init="$watch('useBuiltinAuth', value => {
if(value) {
destClientId = '';
destClientSecret = '';
}
})">
<div class="mb-6">
<label for="use_builtin_auth" class="flex items-center cursor-pointer">
<div class="relative">
<input id="use_builtin_auth" name="use_builtin_auth" type="checkbox"
class="sr-only"
x-model="useBuiltinAuth"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="useBuiltinAuth ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Use rclone's built-in Google authentication (recommended)
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Simple one-click authentication using rclone's shared credentials
</p>
</div>
<div x-bind:class="{ 'opacity-50': useBuiltinAuth }">
<div>
<label for="dest_client_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">
Google Client ID
<span x-show="useBuiltinAuth" class="text-secondary-400 dark:text-secondary-600 text-xs font-normal">(Using rclone default)</span>
</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-id-card text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="text" id="dest_client_id" name="dest_client_id" x-model="destClientId"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
x-bind:disabled="useBuiltinAuth"
placeholder="Google Photos OAuth Client ID" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Client ID from Google Cloud Console
</p>
</div>
<div class="mt-4">
<label for="dest_client_secret" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">
Google Client Secret
<span x-show="useBuiltinAuth" class="text-secondary-400 dark:text-secondary-600 text-xs font-normal">(Using rclone default)</span>
</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-key text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="password" id="dest_client_secret" name="dest_client_secret" x-model="destClientSecret"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
x-bind:disabled="useBuiltinAuth"
placeholder="Google Photos OAuth Client Secret" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Client Secret from Google Cloud Console
</p>
</div>
</div>
<div class="mt-4">
<label for="dest_read_only" class="flex items-center cursor-pointer">
<div class="relative">
<input id="dest_read_only" name="dest_read_only" type="checkbox"
class="sr-only"
x-model="destReadOnly"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="destReadOnly ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Read-only mode
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Only request read-only access to your photos
</p>
</div>
<div class="mt-4">
<label for="dest_start_year" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Start Year (Optional)</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-calendar text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="number" id="dest_start_year" name="dest_start_year" x-model="destStartYear"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Only include photos after this year (default: 2000)" min="1900" max="2100" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Only include photos uploaded after this year
</p>
</div>
<div class="mt-4">
<label for="dest_include_archived" class="flex items-center cursor-pointer">
<div class="relative">
<input id="dest_include_archived" name="dest_include_archived" type="checkbox"
class="sr-only"
x-model="destIncludeArchived"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="destIncludeArchived ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Include archived media
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Include archived photos and videos in media listings
</p>
</div>
<div class="mt-4">
<label for="destination_path" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Album Path</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-images text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="text" id="destination_path" name="destination_path" x-model="destinationPath"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Album path (e.g., /album/my-photos)" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Path within Google Photos where files will be uploaded
</p>
</div>
<div class="p-4 bg-amber-50 rounded-lg border border-amber-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-exclamation-triangle text-amber-500"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-amber-800">Important: Authentication Required</h3>
<div class="mt-2 text-sm text-amber-700">
<p><strong>After saving this configuration</strong>, you will need to authenticate with Google Photos.</p>
<p class="mt-1">The authentication process will require you to:</p>
<ol class="list-decimal list-inside mt-1 space-y-1">
<li>Visit a Google authorization URL</li>
<li>Sign in to your Google account</li>
<li>Grant permission to access your Google Photos</li>
<li>Copy the authorization code back to this application</li>
</ol>
<p class="mt-2 text-xs">
This is a one-time process for each configuration. The application will store your authorization token securely.
</p>
</div>
</div>
</div>
</div>
<div class="p-4 bg-blue-50 rounded-lg border border-blue-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-info-circle text-blue-400"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-blue-800">Authentication Information</h3>
<div class="mt-2 text-sm text-blue-700">
<template x-if="useBuiltinAuth">
<div>
<p>You're using rclone's built-in authentication, which simplifies the setup process:</p>
<ul class="list-disc list-inside mt-1 space-y-1">
<li>No need to create your own Google API credentials</li>
<li>Authentication happens through a browser window</li>
<li>You will need to manually copy the authorization code back</li>
</ul>
<p class="mt-2 text-xs text-amber-600">
<i class="fas fa-exclamation-triangle mr-1"></i>
Note: The built-in authentication uses shared credentials which have rate limits across all rclone users.
If you plan to transfer large amounts of data or run many concurrent transfers, consider creating your own credentials.
</p>
</div>
</template>
<template x-if="!useBuiltinAuth">
<div>
<p>To use Google Photos with your own credentials:</p>
<ol class="list-decimal list-inside mt-1 space-y-1">
<li>Go to the <a href="https://console.cloud.google.com/" target="_blank" class="text-blue-600 underline">Google Cloud Console</a></li>
<li>Create a project and enable the Google Photos API</li>
<li>Create OAuth 2.0 credentials (Client ID & Secret)</li>
<li>Set authorized redirect URI to <code class="bg-blue-100 px-1 py-0.5 rounded">http://localhost:53682/</code></li>
</ol>
</div>
</template>
</div>
</div>
</div>
</div>
<div class="p-4 bg-yellow-50 rounded-lg border border-yellow-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-exclamation-circle text-yellow-500"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-yellow-800">Important Note About Google Photos</h3>
<div class="mt-2 text-sm text-yellow-700">
<p>All media items uploaded to Google Photos with rclone are stored in full resolution at original quality. These uploads will count towards storage in your Google Account.</p>
</div>
</div>
</div>
</div>
</div>
}
+220
View File
@@ -0,0 +1,220 @@
package source
templ GooglePhotosSourceForm() {
<div class="space-y-6" x-init="$watch('useBuiltinAuth', value => {
if(value) {
sourceClientId = '';
sourceClientSecret = '';
}
})">
<div class="mb-6">
<label for="use_builtin_auth" class="flex items-center cursor-pointer">
<div class="relative">
<input id="use_builtin_auth" name="use_builtin_auth" type="checkbox"
class="sr-only"
x-model="useBuiltinAuth"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="useBuiltinAuth ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Use rclone's built-in Google authentication (recommended)
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Simple one-click authentication using rclone's shared credentials
</p>
</div>
<div x-bind:class="{ 'opacity-50': useBuiltinAuth }">
<div>
<label for="source_client_id" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">
Google Client ID
<span x-show="useBuiltinAuth" class="text-secondary-400 dark:text-secondary-600 text-xs font-normal">(Using rclone default)</span>
</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-id-card text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="text" id="source_client_id" name="source_client_id" x-model="sourceClientId"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
x-bind:disabled="useBuiltinAuth"
placeholder="Google Photos OAuth Client ID" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Client ID from Google Cloud Console
</p>
</div>
<div class="mt-4">
<label for="source_client_secret" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">
Google Client Secret
<span x-show="useBuiltinAuth" class="text-secondary-400 dark:text-secondary-600 text-xs font-normal">(Using rclone default)</span>
</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-key text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="password" id="source_client_secret" name="source_client_secret" x-model="sourceClientSecret"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
x-bind:disabled="useBuiltinAuth"
placeholder="Google Photos OAuth Client Secret" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Client Secret from Google Cloud Console
</p>
</div>
</div>
<div class="mt-4">
<label for="source_read_only" class="flex items-center cursor-pointer">
<div class="relative">
<input id="source_read_only" name="source_read_only" type="checkbox"
class="sr-only"
x-model="sourceReadOnly"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="sourceReadOnly ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Read-only mode
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Only request read-only access to your photos
</p>
</div>
<div class="mt-4">
<label for="source_start_year" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Start Year (Optional)</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-calendar text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="number" id="source_start_year" name="source_start_year" x-model="sourceStartYear"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Only include photos after this year (default: 2000)" min="1900" max="2100" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Only include photos uploaded after this year
</p>
</div>
<div class="mt-4">
<label for="source_include_archived" class="flex items-center cursor-pointer">
<div class="relative">
<input id="source_include_archived" name="source_include_archived" type="checkbox"
class="sr-only"
x-model="sourceIncludeArchived"
value="true"
/>
<div class="block bg-gray-200 w-14 h-8 rounded-full"></div>
<div class="dot absolute left-1 top-1 bg-white w-6 h-6 rounded-full transition"
:class="sourceIncludeArchived ? 'transform translate-x-6 bg-primary-500' : ''"></div>
</div>
<div class="ml-3 text-gray-700 font-medium">
Include archived media
</div>
</label>
<p class="mt-1 ml-14 text-xs text-secondary-500 dark:text-secondary-400">
Include archived photos and videos in media listings
</p>
</div>
<div class="mt-4">
<label for="source_path" class="block text-sm font-medium text-secondary-700 dark:text-secondary-300 mb-1">Album Path</label>
<div class="relative">
<div class="absolute inset-y-0 left-0 pl-3 flex items-center pointer-events-none">
<i class="fas fa-images text-secondary-400 dark:text-secondary-600"></i>
</div>
<input type="text" id="source_path" name="source_path" x-model="sourcePath"
class="form-input pl-10 w-full rounded-lg border-secondary-300 dark:border-secondary-700 dark:bg-secondary-800 dark:text-secondary-100 focus:ring-primary-500 focus:border-primary-500"
placeholder="Album path (e.g., /album/my-photos)" />
</div>
<p class="mt-1 text-xs text-secondary-500 dark:text-secondary-400">
Path within Google Photos to download files from
</p>
</div>
<div class="p-4 bg-amber-50 rounded-lg border border-amber-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-exclamation-triangle text-amber-500"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-amber-800">Important: Authentication Required</h3>
<div class="mt-2 text-sm text-amber-700">
<p><strong>After saving this configuration</strong>, you will need to authenticate with Google Photos.</p>
<p class="mt-1">The authentication process will require you to:</p>
<ol class="list-decimal list-inside mt-1 space-y-1">
<li>Visit a Google authorization URL</li>
<li>Sign in to your Google account</li>
<li>Grant permission to access your Google Photos</li>
<li>Copy the authorization code back to this application</li>
</ol>
<p class="mt-2 text-xs">
This is a one-time process for each configuration. The application will store your authorization token securely.
</p>
</div>
</div>
</div>
</div>
<div class="p-4 bg-blue-50 rounded-lg border border-blue-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-info-circle text-blue-400"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-blue-800">Authentication Information</h3>
<div class="mt-2 text-sm text-blue-700">
<template x-if="useBuiltinAuth">
<div>
<p>You're using rclone's built-in authentication, which simplifies the setup process:</p>
<ul class="list-disc list-inside mt-1 space-y-1">
<li>No need to create your own Google API credentials</li>
<li>Authentication happens through a browser window</li>
<li>You will need to manually copy the authorization code back</li>
</ul>
<p class="mt-2 text-xs text-amber-600">
<i class="fas fa-exclamation-triangle mr-1"></i>
Note: The built-in authentication uses shared credentials which have rate limits across all rclone users.
If you plan to transfer large amounts of data or run many concurrent transfers, consider creating your own credentials.
</p>
</div>
</template>
<template x-if="!useBuiltinAuth">
<div>
<p>To use Google Photos with your own credentials:</p>
<ol class="list-decimal list-inside mt-1 space-y-1">
<li>Go to the <a href="https://console.cloud.google.com/" target="_blank" class="text-blue-600 underline">Google Cloud Console</a></li>
<li>Create a project and enable the Google Photos API</li>
<li>Create OAuth 2.0 credentials (Client ID & Secret)</li>
<li>Set authorized redirect URI to <code class="bg-blue-100 px-1 py-0.5 rounded">http://localhost:53682/</code></li>
</ol>
</div>
</template>
</div>
</div>
</div>
</div>
<div class="p-4 bg-yellow-50 rounded-lg border border-yellow-100">
<div class="flex">
<div class="flex-shrink-0">
<i class="fas fa-exclamation-circle text-yellow-500"></i>
</div>
<div class="ml-3">
<h3 class="text-sm font-medium text-yellow-800">Important Note About Google Photos</h3>
<div class="mt-2 text-sm text-yellow-700">
<p>When downloading from Google Photos, be aware that some original metadata may not be preserved. Google Photos processes and may compress some images upon upload.</p>
</div>
</div>
</div>
</div>
</div>
}
+196 -218
View File
@@ -1,11 +1,11 @@
package db
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
"time"
@@ -74,6 +74,10 @@ type TransferConfig struct {
SourceClientSecret string `form:"source_client_secret" gorm:"-"` // Not stored in DB, only used for form
SourceDriveID string `form:"source_drive_id"` // For OneDrive
SourceTeamDrive string `form:"source_team_drive"` // For Google Drive
// Google Photos source fields
SourceReadOnly *bool `form:"source_read_only"` // For Google Photos
SourceStartYear int `form:"source_start_year"` // For Google Photos
SourceIncludeArchived *bool `form:"source_include_archived"` // For Google Photos
// General fields
FilePattern string `gorm:"default:'*'" form:"file_pattern"`
OutputPattern string `form:"output_pattern"` // Pattern for output filenames with date variables
@@ -100,8 +104,13 @@ type TransferConfig struct {
DestClientSecret string `form:"dest_client_secret" gorm:"-"` // Not stored in DB, only used for form
DestDriveID string `form:"dest_drive_id"` // For OneDrive
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
// Google Drive authentication status
GoogleDriveAuthenticated *bool `gorm:"default:false"`
// Google Photos destination fields
DestReadOnly *bool `form:"dest_read_only"` // For Google Photos
DestStartYear int `form:"dest_start_year"` // For Google Photos
DestIncludeArchived *bool `form:"dest_include_archived"` // For Google Photos
// Security fields
UseBuiltinAuth *bool `form:"use_builtin_auth"` // For Google and other OAuth services
GoogleDriveAuthenticated *bool // Whether Google Drive auth is completed
// General fields
ArchivePath string `form:"archive_path"`
ArchiveEnabled *bool `gorm:"default:false" form:"archive_enabled"`
@@ -632,6 +641,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
args = append(args, "team_drive", config.SourceTeamDrive)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
}
case "gphotos":
args := []string{
"config", "create", sourceName, "google photos",
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
// Only add client_id and client_secret if they're provided (not empty)
// This allows using rclone's built-in authentication
if config.SourceClientID != "" && config.SourceClientSecret != "" {
args = append(args, "client_id", config.SourceClientID)
args = append(args, "client_secret", config.SourceClientSecret)
}
// Add read_only option if specified
if config.SourceReadOnly != nil && *config.SourceReadOnly {
args = append(args, "read_only", "true")
}
// Add start_year if specified
if config.SourceStartYear > 0 {
args = append(args, "start_year", strconv.Itoa(config.SourceStartYear))
}
// Add include_archived if specified
if config.SourceIncludeArchived != nil && *config.SourceIncludeArchived {
args = append(args, "include_archived", "true")
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
@@ -829,6 +872,40 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
args = append(args, "root_folder_id", config.DestDriveID)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
}
case "gphotos":
args := []string{
"config", "create", destName, "google photos",
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
// Only add client_id and client_secret if they're provided (not empty)
// This allows using rclone's built-in authentication
if config.DestClientID != "" && config.DestClientSecret != "" {
args = append(args, "client_id", config.DestClientID)
args = append(args, "client_secret", config.DestClientSecret)
}
// Add read_only option if specified
if config.DestReadOnly != nil && *config.DestReadOnly {
args = append(args, "read_only", "true")
}
// Add start_year if specified
if config.DestStartYear > 0 {
args = append(args, "start_year", strconv.Itoa(config.DestStartYear))
}
// Add include_archived if specified
if config.DestIncludeArchived != nil && *config.DestIncludeArchived {
args = append(args, "include_archived", "true")
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config: %v\nOutput: %s", err, output)
@@ -1010,235 +1087,129 @@ func (db *DB) StoreGoogleDriveToken(configIDStr string, token string) error {
// GenerateRcloneConfigWithToken generates a rclone config file for a transfer config with a provided token
func (db *DB) GenerateRcloneConfigWithToken(config *TransferConfig, token string) error {
// Ensure we have a config directory
dataDir := os.Getenv("DATA_DIR")
if dataDir == "" {
dataDir = "./data"
}
configDir := filepath.Join(dataDir, "configs")
if err := os.MkdirAll(configDir, 0755); err != nil {
return err
// Get the config path
configPath := db.GetConfigRclonePath(config)
if configPath == "" {
return fmt.Errorf("failed to get config path")
}
// Generate rclone config based on the config type
configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", config.ID))
// Clean up the token to ensure it's a single line JSON
token = strings.TrimSpace(token)
token = strings.ReplaceAll(token, "\n", "")
token = strings.ReplaceAll(token, "\r", "")
// Create a new config content
var configContent strings.Builder
// Determine if this is a source or destination config
var configType, section, clientID, clientSecret string
var readOnly, includeArchived *bool
var startYear int
// First add the source configuration
sourceName := fmt.Sprintf("source_%d", config.ID)
// Handle the source configuration based on type
switch config.SourceType {
case "google_drive":
// Create a Google Drive remote using the "source_ID" naming convention for sources
sourceSection := fmt.Sprintf("[%s]\ntype = drive\n", sourceName)
// Add custom client ID and secret if provided
if config.SourceClientID != "" && config.SourceClientSecret != "" {
sourceSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n",
config.SourceClientID, config.SourceClientSecret)
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
configType = config.DestinationType
section = "dest"
clientID = config.DestClientID
clientSecret = config.DestClientSecret
readOnly = config.DestReadOnly
startYear = config.DestStartYear
includeArchived = config.DestIncludeArchived
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
configType = config.SourceType
section = "source"
clientID = config.SourceClientID
clientSecret = config.SourceClientSecret
readOnly = config.SourceReadOnly
startYear = config.SourceStartYear
includeArchived = config.SourceIncludeArchived
} else {
return fmt.Errorf("config is not for Google Drive or Google Photos")
}
// Read the existing config
content, err := os.ReadFile(configPath)
if err != nil {
return fmt.Errorf("failed to read config file: %v", err)
}
// Prepare the section content
var sectionContent string
if configType == "gdrive" {
sectionContent = fmt.Sprintf("[%s_%d]\ntype = drive\n", section, config.ID)
if clientID != "" {
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
}
if clientSecret != "" {
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
}
sectionContent += fmt.Sprintf("token = %s\n", token)
// Add team drive if specified
if config.SourceTeamDrive != "" {
sourceSection += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive)
if section == "source" && config.SourceTeamDrive != "" {
sectionContent += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive)
} else if section == "dest" && config.DestTeamDrive != "" {
sectionContent += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive)
}
// Clean up token string to prevent syntax errors and ensure it's a single line JSON
// First, remove any whitespace from the beginning and end
cleanToken := strings.TrimSpace(token)
// Add read-only flag if specified
if readOnly != nil && *readOnly {
sectionContent += "read_only = true\n"
}
} else if configType == "gphotos" {
sectionContent = fmt.Sprintf("[%s_%d]\ntype = google photos\n", section, config.ID)
if clientID != "" {
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
}
if clientSecret != "" {
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
}
sectionContent += fmt.Sprintf("token = %s\n", token)
// Check if it's already a JSON object
if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") {
// It's a JSON object, but we need to make sure it's a single line
var jsonObj map[string]interface{}
if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil {
// Successfully parsed the JSON, now re-marshal it as a compact single line
compactJSON, err := json.Marshal(jsonObj)
if err == nil {
// Use the compact JSON as the token
sourceSection += fmt.Sprintf("token = %s\n", string(compactJSON))
} else {
// If there was an error re-marshaling, use the original but remove newlines
// Replace all newlines and carriage returns with empty string
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
}
} else {
// If we couldn't parse the JSON, just remove newlines and carriage returns
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
}
} else {
// Not a valid JSON, try to fix it
// First ensure it starts and ends with braces
if !strings.HasPrefix(cleanToken, "{") {
cleanToken = "{" + cleanToken
}
if !strings.HasSuffix(cleanToken, "}") {
cleanToken = cleanToken + "}"
}
// Remove all newlines and carriage returns
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
sourceSection += fmt.Sprintf("token = %s\n", singleLineToken)
// Add read-only flag if specified
if readOnly != nil && *readOnly {
sectionContent += "read_only = true\n"
}
configContent.WriteString(sourceSection)
configContent.WriteString("\n")
// Add start year if specified
if startYear > 0 {
sectionContent += fmt.Sprintf("start_year = %d\n", startYear)
}
case "sftp", "s3", "minio", "b2", "smb", "ftp", "webdav", "nextcloud", "onedrive":
// For complex source types, use the GenerateRcloneConfig function
// to create a temporary file, read it, and then append that content
tempDir, err := os.MkdirTemp("", "gomft_temp")
// Add include_archived flag if specified and true
if includeArchived != nil && *includeArchived {
sectionContent += "include_archived = true\n"
}
}
// Find the section in the existing config
sectionPattern := regexp.MustCompile(fmt.Sprintf(`\[%s_%d\][^\[]*`, section, config.ID))
if sectionPattern.MatchString(string(content)) {
// Replace the existing section
newContent := sectionPattern.ReplaceAllString(string(content), sectionContent)
err = os.WriteFile(configPath, []byte(newContent), 0644)
if err != nil {
return fmt.Errorf("failed to create temp directory: %v", err)
return fmt.Errorf("failed to write updated config file: %v", err)
}
defer os.RemoveAll(tempDir)
tempConfigPath := filepath.Join(tempDir, "temp_config.conf")
// Create a temporary file with just the source configuration
tempContent := fmt.Sprintf("[%s]\ntype = local\n", sourceName)
if err := os.WriteFile(tempConfigPath, []byte(tempContent), 0600); err != nil {
return fmt.Errorf("failed to write temporary config: %v", err)
}
// Get rclone path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
rclonePath = "rclone"
}
// Use the appropriate rclone command to configure the source
var args []string
switch config.SourceType {
case "sftp":
args = []string{
"config", "create", sourceName, "sftp",
"host", config.SourceHost,
"user", config.SourceUser,
"port", fmt.Sprintf("%d", config.SourcePort),
"--non-interactive",
"--config", tempConfigPath,
"--log-level", "ERROR",
}
if config.SourcePassword != "" {
args = append(args, "pass", config.SourcePassword)
}
if config.SourceKeyFile != "" {
args = append(args, "key_file", config.SourceKeyFile)
}
case "s3":
args = []string{
"config", "create", sourceName, "s3",
"provider", "AWS",
"env_auth", "false",
"access_key_id", config.SourceAccessKey,
"secret_access_key", config.SourceSecretKey,
"region", config.SourceRegion,
"--non-interactive",
"--config", tempConfigPath,
"--log-level", "ERROR",
}
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
}
// If we have arguments, execute the command
if len(args) > 0 {
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config: %v\nOutput: %s", err, output)
}
// Read the generated config
sourceConfig, err := os.ReadFile(tempConfigPath)
} else {
// Append the section to the config
file, err := os.OpenFile(configPath, os.O_APPEND|os.O_WRONLY, 0644)
if err != nil {
return fmt.Errorf("failed to read temporary config: %v", err)
return fmt.Errorf("failed to open config file for appending: %v", err)
}
defer file.Close()
_, err = file.WriteString("\n" + sectionContent)
if err != nil {
return fmt.Errorf("failed to append to config file: %v", err)
}
}
// Add it to our config content
configContent.WriteString(string(sourceConfig))
configContent.WriteString("\n")
}
default:
// For local source or other simple types
sourceSection := fmt.Sprintf("[%s]\ntype = local\n\n", sourceName)
configContent.WriteString(sourceSection)
// Update the authentication status
authenticated := true
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
config.SetGoogleAuthenticated(authenticated)
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
config.SetGoogleAuthenticated(authenticated)
}
// Now add the destination configuration
destName := fmt.Sprintf("dest_%d", config.ID)
// Set up the destination section (only supporting Google Drive for now)
if config.DestinationType == "gdrive" || config.DestinationType == "google_drive" {
// Create a Google Drive remote using "dest_ID" naming convention
destSection := fmt.Sprintf("[%s]\ntype = drive\n", destName)
// Add custom client ID and secret if provided
if config.DestClientID != "" && config.DestClientSecret != "" {
destSection += fmt.Sprintf("client_id = %s\nclient_secret = %s\n",
config.DestClientID, config.DestClientSecret)
}
// Clean up token string to prevent syntax errors and ensure it's a single line JSON
// First, remove any whitespace from the beginning and end
cleanToken := strings.TrimSpace(token)
// Check if it's already a JSON object
if strings.HasPrefix(cleanToken, "{") && strings.HasSuffix(cleanToken, "}") {
// It's a JSON object, but we need to make sure it's a single line
var jsonObj map[string]interface{}
if err := json.Unmarshal([]byte(cleanToken), &jsonObj); err == nil {
// Successfully parsed the JSON, now re-marshal it as a compact single line
compactJSON, err := json.Marshal(jsonObj)
if err == nil {
// Use the compact JSON as the token
destSection += fmt.Sprintf("token = %s\n", string(compactJSON))
} else {
// If there was an error re-marshaling, use the original but remove newlines
// Replace all newlines and carriage returns with empty string
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
}
} else {
// If we couldn't parse the JSON, just remove newlines and carriage returns
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
}
} else {
// Not a valid JSON, try to fix it
// First ensure it starts and ends with braces
if !strings.HasPrefix(cleanToken, "{") {
cleanToken = "{" + cleanToken
}
if !strings.HasSuffix(cleanToken, "}") {
cleanToken = cleanToken + "}"
}
// Remove all newlines and carriage returns
singleLineToken := strings.ReplaceAll(cleanToken, "\n", "")
singleLineToken = strings.ReplaceAll(singleLineToken, "\r", "")
destSection += fmt.Sprintf("token = %s\n", singleLineToken)
}
// Add to the config content
configContent.WriteString(destSection)
} else {
// Add a simple local destination for testing or if no specific destination type is handled
destSection := fmt.Sprintf("[%s]\ntype = local\n", destName)
configContent.WriteString(destSection)
}
// Write the config file
return os.WriteFile(configPath, []byte(configContent.String()), 0644)
return nil
}
// GetIsAdmin returns the value of IsAdmin with a default if nil
@@ -1306,19 +1277,26 @@ func (tc *TransferConfig) SetDestPassiveMode(value bool) {
tc.DestPassiveMode = &value
}
// GetGoogleDriveAuthenticated returns the value of GoogleDriveAuthenticated with a default if nil
// GetGoogleDriveAuthenticated returns whether the transfer config has been authenticated with Google Drive
func (tc *TransferConfig) GetGoogleDriveAuthenticated() bool {
if tc.GoogleDriveAuthenticated == nil {
return false // Default to false if not set
}
return *tc.GoogleDriveAuthenticated
return tc.GoogleDriveAuthenticated != nil && *tc.GoogleDriveAuthenticated
}
// SetGoogleDriveAuthenticated sets the GoogleDriveAuthenticated field
// SetGoogleDriveAuthenticated sets the Google Drive authentication status
func (tc *TransferConfig) SetGoogleDriveAuthenticated(value bool) {
tc.GoogleDriveAuthenticated = &value
}
// GetGoogleAuthenticated is an alias for GetGoogleDriveAuthenticated for better semantics when working with Google Photos
func (tc *TransferConfig) GetGoogleAuthenticated() bool {
return tc.GetGoogleDriveAuthenticated()
}
// SetGoogleAuthenticated is an alias for SetGoogleDriveAuthenticated for better semantics when working with Google Photos
func (tc *TransferConfig) SetGoogleAuthenticated(value bool) {
tc.SetGoogleDriveAuthenticated(value)
}
// GetArchiveEnabled returns the value of ArchiveEnabled with a default if nil
func (tc *TransferConfig) GetArchiveEnabled() bool {
if tc.ArchiveEnabled == nil {
+342
View File
@@ -1416,3 +1416,345 @@ func TestGoogleDriveTeamDrive(t *testing.T) {
err = db.DeleteTransferConfig(teamDriveBothConfig.ID)
assert.NoError(t, err)
}
func TestGooglePhotosTransferConfig(t *testing.T) {
db := setupTestDB(t)
// Create a test user
testUser := &User{
Email: fmt.Sprintf("gphotos-test-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hashed_password",
LastPasswordChange: time.Now(),
}
err := db.CreateUser(testUser)
if err != nil {
t.Fatalf("Failed to create user: %v", err)
}
// Test 1: Create config with Google Photos as source
sourceReadOnly := true
sourceIncludeArchived := false
useBuiltinAuth := true
googleSourceConfig := &TransferConfig{
Name: "Google Photos Source Test",
SourceType: "gphotos",
SourcePath: "/albums/vacation",
SourceClientID: "google_client_id",
SourceClientSecret: "google_client_secret",
SourceReadOnly: &sourceReadOnly,
SourceStartYear: 2015,
SourceIncludeArchived: &sourceIncludeArchived,
UseBuiltinAuth: &useBuiltinAuth,
DestinationType: "local",
DestinationPath: "/local/destination/path",
FilePattern: "*.jpg",
CreatedBy: testUser.ID,
}
// Set authenticated status
authenticated := true
googleSourceConfig.GoogleDriveAuthenticated = &authenticated
// Create the config
err = db.CreateTransferConfig(googleSourceConfig)
assert.NoError(t, err)
assert.NotZero(t, googleSourceConfig.ID, "Config ID should be set after creation")
// Test 2: Create config with Google Photos as destination
destReadOnly := false
destIncludeArchived := true
googleDestConfig := &TransferConfig{
Name: "Google Photos Destination Test",
SourceType: "local",
SourcePath: "/local/source/path",
DestinationType: "gphotos",
DestinationPath: "/albums/upload",
DestClientID: "google_client_id",
DestClientSecret: "google_client_secret",
DestReadOnly: &destReadOnly,
DestStartYear: 2018,
DestIncludeArchived: &destIncludeArchived,
UseBuiltinAuth: &useBuiltinAuth,
FilePattern: "*.png",
CreatedBy: testUser.ID,
}
// Set authenticated status
googleDestConfig.GoogleDriveAuthenticated = &authenticated
// Create the config
err = db.CreateTransferConfig(googleDestConfig)
assert.NoError(t, err)
assert.NotZero(t, googleDestConfig.ID, "Config ID should be set after creation")
// Test 3: Create config with Google Photos as both source and destination
googleBothConfig := &TransferConfig{
Name: "Google Photos Both Test",
SourceType: "gphotos",
SourcePath: "/albums/source_album",
SourceClientID: "source_client_id",
SourceClientSecret: "source_client_secret",
SourceReadOnly: &sourceReadOnly,
SourceStartYear: 2020,
SourceIncludeArchived: &sourceIncludeArchived,
DestinationType: "gphotos",
DestinationPath: "/albums/dest_album",
DestClientID: "dest_client_id",
DestClientSecret: "dest_client_secret",
DestReadOnly: &destReadOnly,
DestStartYear: 2020,
DestIncludeArchived: &destIncludeArchived,
UseBuiltinAuth: &useBuiltinAuth,
FilePattern: "*.jpeg",
CreatedBy: testUser.ID,
}
// Set authenticated status
googleBothConfig.GoogleDriveAuthenticated = &authenticated
// Create the config
err = db.CreateTransferConfig(googleBothConfig)
assert.NoError(t, err)
assert.NotZero(t, googleBothConfig.ID, "Config ID should be set after creation")
// Verify configs were created properly
retrievedConfig, err := db.GetTransferConfig(googleSourceConfig.ID)
assert.NoError(t, err)
assert.Equal(t, "gphotos", retrievedConfig.SourceType)
assert.Equal(t, sourceReadOnly, *retrievedConfig.SourceReadOnly)
assert.Equal(t, 2015, retrievedConfig.SourceStartYear)
assert.Equal(t, sourceIncludeArchived, *retrievedConfig.SourceIncludeArchived)
assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth)
assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated())
retrievedConfig, err = db.GetTransferConfig(googleDestConfig.ID)
assert.NoError(t, err)
assert.Equal(t, "gphotos", retrievedConfig.DestinationType)
assert.Equal(t, destReadOnly, *retrievedConfig.DestReadOnly)
assert.Equal(t, 2018, retrievedConfig.DestStartYear)
assert.Equal(t, destIncludeArchived, *retrievedConfig.DestIncludeArchived)
assert.Equal(t, useBuiltinAuth, *retrievedConfig.UseBuiltinAuth)
assert.Equal(t, true, retrievedConfig.GetGoogleAuthenticated())
}
func TestGooglePhotosRcloneConfig(t *testing.T) {
// Create a temporary test directory
tempDir, err := os.MkdirTemp("", "gomft-test")
if err != nil {
t.Fatalf("Failed to create temp directory: %v", err)
}
defer os.RemoveAll(tempDir)
// Set up data directory
dataDir := filepath.Join(tempDir, "data")
configDir := filepath.Join(dataDir, "configs")
err = os.MkdirAll(configDir, 0755)
if err != nil {
t.Fatalf("Failed to create config directory: %v", err)
}
// Set DATA_DIR environment variable
oldDataDir := os.Getenv("DATA_DIR")
defer os.Setenv("DATA_DIR", oldDataDir)
os.Setenv("DATA_DIR", dataDir)
// Initialize test database
db := setupTestDB(t)
// Create a test user
testUser := &User{
Email: fmt.Sprintf("gphotos-rclone-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hashed_password",
LastPasswordChange: time.Now(),
}
err = db.CreateUser(testUser)
assert.NoError(t, err)
// TEST 1: Google Photos as source with standard options
sourceReadOnly := true
sourceIncludeArchived := false
useBuiltinAuth := true
gphotosSourceConfig := &TransferConfig{
ID: 1, // Force ID for predictable config path
Name: "Google Photos Source Config",
SourceType: "gphotos",
SourcePath: "/albums/vacation",
SourceClientID: "test_client_id",
SourceClientSecret: "test_client_secret",
SourceReadOnly: &sourceReadOnly,
SourceStartYear: 2015,
SourceIncludeArchived: &sourceIncludeArchived,
UseBuiltinAuth: &useBuiltinAuth,
DestinationType: "local",
DestinationPath: "/tmp/destination",
FilePattern: "*.jpg",
CreatedBy: testUser.ID,
}
// Generate rclone config
err = db.GenerateRcloneConfig(gphotosSourceConfig)
assert.NoError(t, err)
// Check if config file exists
configPath := filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosSourceConfig.ID))
_, err = os.Stat(configPath)
assert.NoError(t, err, "Config file should exist")
// Read config file content
content, err := os.ReadFile(configPath)
assert.NoError(t, err)
configContent := string(content)
// Check for Google Photos source section
assert.Contains(t, configContent, "[source_1]")
assert.Contains(t, configContent, "type = google photos")
assert.Contains(t, configContent, "client_id = test_client_id")
assert.Contains(t, configContent, "client_secret = test_client_secret")
assert.Contains(t, configContent, "read_only = true")
assert.Contains(t, configContent, "start_year = 2015")
assert.NotContains(t, configContent, "include_archived = true") // This should be false and not included
// TEST 2: Google Photos as destination with authenticated token
destReadOnly := false
destIncludeArchived := true
gphotosDestConfig := &TransferConfig{
ID: 2, // Force ID for predictable config path
Name: "Google Photos Destination Config",
SourceType: "local",
SourcePath: "/tmp/source",
DestinationType: "gphotos",
DestinationPath: "/albums/upload",
DestClientID: "dest_client_id",
DestClientSecret: "dest_client_secret",
DestReadOnly: &destReadOnly,
DestStartYear: 2018,
DestIncludeArchived: &destIncludeArchived,
UseBuiltinAuth: &useBuiltinAuth,
FilePattern: "*.png",
CreatedBy: testUser.ID,
}
// Set authentication status
authenticated := true
gphotosDestConfig.GoogleDriveAuthenticated = &authenticated
// Generate config first (needed for token update)
err = db.GenerateRcloneConfig(gphotosDestConfig)
assert.NoError(t, err)
// Now test token handling with GenerateRcloneConfigWithToken
testToken := `{"access_token":"test-token","token_type":"Bearer","refresh_token":"test-refresh","expiry":"2023-12-31T23:59:59Z"}`
err = db.GenerateRcloneConfigWithToken(gphotosDestConfig, testToken)
assert.NoError(t, err)
// Check updated config
configPath = filepath.Join(configDir, fmt.Sprintf("config_%d.conf", gphotosDestConfig.ID))
_, err = os.Stat(configPath)
assert.NoError(t, err, "Config file should exist")
// Read config file content
content, err = os.ReadFile(configPath)
assert.NoError(t, err)
configContent = string(content)
// Check for Google Photos destination section with token
assert.Contains(t, configContent, "type = google photos")
assert.Contains(t, configContent, "client_id = dest_client_id")
assert.Contains(t, configContent, "client_secret = dest_client_secret")
assert.Contains(t, configContent, "token = {")
assert.Contains(t, configContent, "access_token")
assert.Contains(t, configContent, "test-token")
assert.Contains(t, configContent, "refresh_token")
assert.Contains(t, configContent, "test-refresh")
assert.Contains(t, configContent, "include_archived = true")
assert.NotContains(t, configContent, "read_only = false") // This should be false and not included
}
func TestGooglePhotosAuthentication(t *testing.T) {
// Initialize test database
db := setupTestDB(t)
// Create a test user
testUser := &User{
Email: fmt.Sprintf("gphotos-auth-%d@example.com", time.Now().UnixNano()),
PasswordHash: "hashed_password",
LastPasswordChange: time.Now(),
}
err := db.CreateUser(testUser)
assert.NoError(t, err)
// Create a transfer config with Google Photos
readOnly := true
includeArchived := false
useBuiltinAuth := true
gPhotosConfig := &TransferConfig{
Name: "Test Google Photos Auth",
SourceType: "gphotos",
SourcePath: "/albums/vacation",
SourceClientID: "test_client_id",
SourceClientSecret: "test_client_secret",
SourceReadOnly: &readOnly,
SourceStartYear: 2015,
SourceIncludeArchived: &includeArchived,
UseBuiltinAuth: &useBuiltinAuth,
DestinationType: "local",
DestinationPath: "/tmp/destination",
FilePattern: "*.jpg",
CreatedBy: testUser.ID,
}
// Create the config
err = db.CreateTransferConfig(gPhotosConfig)
assert.NoError(t, err)
assert.NotZero(t, gPhotosConfig.ID)
// Test initial authentication state
// Should be false when first created
authenticated := gPhotosConfig.GetGoogleAuthenticated()
assert.False(t, authenticated)
t.Logf("Initial GoogleDriveAuthenticated value: %v", gPhotosConfig.GoogleDriveAuthenticated)
// Test generic Google authentication method (new)
gPhotosConfig.SetGoogleAuthenticated(true)
t.Logf("After SetGoogleAuthenticated(true): %v", gPhotosConfig.GoogleDriveAuthenticated)
// Save the updated config to the database
err = db.UpdateTransferConfig(gPhotosConfig)
assert.NoError(t, err)
t.Logf("After UpdateTransferConfig: %v", gPhotosConfig.GoogleDriveAuthenticated)
// Verify authentication status is updated
updatedConfig, err := db.GetTransferConfig(gPhotosConfig.ID)
assert.NoError(t, err)
t.Logf("Retrieved config GoogleDriveAuthenticated: %v", updatedConfig.GoogleDriveAuthenticated)
assert.True(t, updatedConfig.GetGoogleAuthenticated())
// Verify it can be unset
updatedConfig.SetGoogleAuthenticated(false)
t.Logf("After SetGoogleAuthenticated(false): %v", updatedConfig.GoogleDriveAuthenticated)
// Save the updated config to the database
err = db.UpdateTransferConfig(updatedConfig)
assert.NoError(t, err)
// Verify authentication status is updated
updatedConfig2, err := db.GetTransferConfig(gPhotosConfig.ID)
assert.NoError(t, err)
t.Logf("Retrieved config2 GoogleDriveAuthenticated: %v", updatedConfig2.GoogleDriveAuthenticated)
assert.False(t, updatedConfig2.GetGoogleAuthenticated())
// Test with the old method naming for backward compatibility
updatedConfig2.SetGoogleDriveAuthenticated(true)
t.Logf("After SetGoogleDriveAuthenticated(true): %v", updatedConfig2.GoogleDriveAuthenticated)
// Save the updated config to the database
err = db.UpdateTransferConfig(updatedConfig2)
assert.NoError(t, err)
// Verify authentication status is updated when using the old method
updatedConfig3, err := db.GetTransferConfig(gPhotosConfig.ID)
assert.NoError(t, err)
t.Logf("Retrieved config3 GoogleDriveAuthenticated: %v", updatedConfig3.GoogleDriveAuthenticated)
assert.True(t, updatedConfig3.GetGoogleAuthenticated())
assert.True(t, updatedConfig3.GetGoogleDriveAuthenticated())
}
@@ -0,0 +1,61 @@
package migrations
import (
"github.com/go-gormigrate/gormigrate/v2"
"gorm.io/gorm"
)
// AddGooglePhotosSupport adds Google Photos related fields to the transfer_configs table
func AddGooglePhotosSupport() *gormigrate.Migration {
return &gormigrate.Migration{
ID: "20240518_add_google_photos_support",
Migrate: func(tx *gorm.DB) error {
// Add Google Photos source fields
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_read_only BOOLEAN DEFAULT false").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_start_year INTEGER DEFAULT 0").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN source_include_archived BOOLEAN DEFAULT false").Error; err != nil {
return err
}
// Add Google Photos destination fields
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_read_only BOOLEAN DEFAULT false").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_start_year INTEGER DEFAULT 0").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs ADD COLUMN dest_include_archived BOOLEAN DEFAULT false").Error; err != nil {
return err
}
// Add OAuth field
return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN use_builtin_auth BOOLEAN DEFAULT true").Error
},
Rollback: func(tx *gorm.DB) error {
// Remove all added columns in reverse order
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN use_builtin_auth").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_include_archived").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_start_year").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN dest_read_only").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_include_archived").Error; err != nil {
return err
}
if err := tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_start_year").Error; err != nil {
return err
}
return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN source_read_only").Error
},
}
}
+1
View File
@@ -17,6 +17,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
UpdateSkipProcessedFilesToNullable(),
AddWebhookSupport(),
AddGoogleDriveAuthenticated(),
AddGooglePhotosSupport(),
}
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
+6
View File
@@ -535,6 +535,12 @@ func (s *Scheduler) executeConfigTransfer(job db.Job, config db.TransferConfig,
if maxConcurrent < 1 {
maxConcurrent = 1 // Default to 1 if not set
}
// Limit Google Photos to 1 concurrent transfers
if config.SourceType == "gphotos" || config.DestinationType == "gphotos" {
maxConcurrent = 1
}
s.log.LogInfo("Using %d concurrent transfers for job %d, config %d", maxConcurrent, job.ID, config.ID)
// Create wait group for concurrent processing
+42
View File
@@ -101,6 +101,27 @@ func (h *Handlers) HandleCreateConfig(c *gin.Context) {
destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true"
config.DestPassiveMode = &destPassiveModeValue
// Google Photos specific fields
destReadOnlyVal := c.Request.FormValue("dest_read_only")
destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true"
config.DestReadOnly = &destReadOnlyValue
sourceReadOnlyVal := c.Request.FormValue("source_read_only")
sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true"
config.SourceReadOnly = &sourceReadOnlyValue
destIncludeArchivedVal := c.Request.FormValue("dest_include_archived")
destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true"
config.DestIncludeArchived = &destIncludeArchivedValue
sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived")
sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true"
config.SourceIncludeArchived = &sourceIncludeArchivedValue
useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth")
useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true"
config.UseBuiltinAuth = &useBuiltinAuthValue
if err := h.DB.Create(&config).Error; err != nil {
log.Printf("Error creating config: %v", err)
c.String(http.StatusInternalServerError, fmt.Sprintf("Failed to create config: %v", err))
@@ -171,6 +192,27 @@ func (h *Handlers) HandleUpdateConfig(c *gin.Context) {
destPassiveModeValue := destPassiveModeVal == "on" || destPassiveModeVal == "true"
config.DestPassiveMode = &destPassiveModeValue
// Google Photos specific fields
destReadOnlyVal := c.Request.FormValue("dest_read_only")
destReadOnlyValue := destReadOnlyVal == "on" || destReadOnlyVal == "true"
config.DestReadOnly = &destReadOnlyValue
sourceReadOnlyVal := c.Request.FormValue("source_read_only")
sourceReadOnlyValue := sourceReadOnlyVal == "on" || sourceReadOnlyVal == "true"
config.SourceReadOnly = &sourceReadOnlyValue
destIncludeArchivedVal := c.Request.FormValue("dest_include_archived")
destIncludeArchivedValue := destIncludeArchivedVal == "on" || destIncludeArchivedVal == "true"
config.DestIncludeArchived = &destIncludeArchivedValue
sourceIncludeArchivedVal := c.Request.FormValue("source_include_archived")
sourceIncludeArchivedValue := sourceIncludeArchivedVal == "on" || sourceIncludeArchivedVal == "true"
config.SourceIncludeArchived = &sourceIncludeArchivedValue
useBuiltinAuthVal := c.Request.FormValue("use_builtin_auth")
useBuiltinAuthValue := useBuiltinAuthVal == "on" || useBuiltinAuthVal == "true"
config.UseBuiltinAuth = &useBuiltinAuthValue
// Preserve fields that shouldn't be updated
config.CreatedBy = oldConfig.CreatedBy
+36 -16
View File
@@ -36,9 +36,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) {
return
}
// Ensure it's a Google Drive configuration
if config.DestinationType != "gdrive" {
RenderErrorPage(c, "Not a Google Drive configuration", "The selected configuration is not set up for Google Drive")
// Ensure it's a Google Drive or Google Photos configuration
if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" {
RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos")
return
}
@@ -81,9 +81,9 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) {
// Define the redirect URI for our callback
redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL)
// Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV
clientID := os.Getenv("GDRIVE_CLIENT_ID")
clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET")
// Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV
clientID := os.Getenv("GOOGLE_CLIENT_ID")
clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET")
if clientID == "" || clientSecret == "" {
// Check if we have client credentials in the existing config file
@@ -109,7 +109,7 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) {
clientSecret = existingClientSecret
} else {
// If we still can't find a matching secret, show an error
RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.")
RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.")
return
}
}
@@ -123,13 +123,28 @@ func (h *Handlers) HandleGDriveAuth(c *gin.Context) {
// Store config ID in cookie for use during callback
c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true)
// Determine the appropriate scope based on destination type
var scope string
if config.DestinationType == "gphotos" {
// Read-only access is handled elsewhere in the config; here we need the full auth scope
scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary")
} else {
// Default to Google Drive scope
scope = url.QueryEscape("https://www.googleapis.com/auth/drive")
}
// Create a config file with redirect URI-based auth
configContent := fmt.Sprintf(`[temp_gdrive]
type = drive
configType := "drive"
if config.DestinationType == "gphotos" {
configType = "google photos"
}
configContent := fmt.Sprintf(`[temp_%s]
type = %s
client_id = %s
client_secret = %s
redirect_url = %s
`, clientID, clientSecret, redirectURI)
`, config.DestinationType, configType, clientID, clientSecret, redirectURI)
// Write the config file
if err := os.WriteFile(tempConfigPath, []byte(configContent), 0644); err != nil {
@@ -138,7 +153,6 @@ redirect_url = %s
}
// Direct Google OAuth URL with our redirect
scope := url.QueryEscape("https://www.googleapis.com/auth/drive")
authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s",
url.QueryEscape(clientID),
url.QueryEscape(redirectURI),
@@ -205,9 +219,9 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) {
return
}
// Attempt to get GDRIVE_CLIENT_ID and GDRIVE_CLIENT_SECRET from ENV
clientID := os.Getenv("GDRIVE_CLIENT_ID")
clientSecret := os.Getenv("GDRIVE_CLIENT_SECRET")
// Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV
clientID := os.Getenv("GOOGLE_CLIENT_ID")
clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET")
if clientID == "" || clientSecret == "" {
// Check if we have client credentials in the existing config file
@@ -233,7 +247,7 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) {
clientSecret = existingClientSecret
} else {
// If we still can't find a matching secret, show an error
RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google Drive authentication.")
RenderErrorPage(c, "Missing client secret", "You provided a custom client ID but no client secret. Both are required for Google authentication.")
return
}
}
@@ -313,7 +327,13 @@ func (h *Handlers) HandleGDriveAuthCallback(c *gin.Context) {
c.SetCookie("gdrive_config_id", "", -1, "/", "", false, true)
// Redirect to the config list with a success message
c.Redirect(http.StatusFound, "/configs?status=gdrive_auth_success")
var successParam string
if config.DestinationType == "gphotos" {
successParam = "gphotos_auth_success"
} else {
successParam = "gdrive_auth_success"
}
c.Redirect(http.StatusFound, fmt.Sprintf("/configs?status=%s", successParam))
}
// HandleGDriveTokenProcess processes a Google Drive token directly from a URL parameter
@@ -0,0 +1,429 @@
package handlers
import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strconv"
"testing"
"time"
"github.com/gin-gonic/gin"
"github.com/starfleetcptn/gomft/internal/db"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
// DBInterface defines the methods we need to mock for our tests
type DBInterface interface {
GetTransferConfig(id uint) (*db.TransferConfig, error)
GetConfigRclonePath(config *db.TransferConfig) string
GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error
GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string)
}
// MockDB is a mock implementation of the DB interface for testing
type MockDB struct {
mock.Mock
}
// Implement the necessary methods from the DB interface for our tests
func (m *MockDB) GetTransferConfig(id uint) (*db.TransferConfig, error) {
args := m.Called(id)
if args.Get(0) == nil {
return nil, args.Error(1)
}
return args.Get(0).(*db.TransferConfig), args.Error(1)
}
func (m *MockDB) GetConfigRclonePath(config *db.TransferConfig) string {
args := m.Called(config)
return args.String(0)
}
func (m *MockDB) GenerateRcloneConfigWithToken(config *db.TransferConfig, token string) error {
args := m.Called(config, token)
return args.Error(0)
}
func (m *MockDB) GetGDriveCredentialsFromConfig(config *db.TransferConfig) (string, string) {
args := m.Called(config)
return args.String(0), args.String(1)
}
// MockHandlers is a modified version of Handlers that accepts our mock DB
type MockHandlers struct {
DB DBInterface
}
// HandleGDriveAuth is a copy of the original method but using our interface
func (h *MockHandlers) HandleGDriveAuth(c *gin.Context) {
// Get the config ID from the query parameter
configIDStr := c.Param("id")
if configIDStr == "" {
RenderErrorPage(c, "Missing configuration ID", "")
return
}
configID, err := strconv.ParseUint(configIDStr, 10, 64)
if err != nil {
RenderErrorPage(c, "Invalid configuration ID", err.Error())
return
}
// Get the configuration
config, err := h.DB.GetTransferConfig(uint(configID))
if err != nil {
RenderErrorPage(c, "Configuration not found", err.Error())
return
}
// Ensure it's a Google Drive or Google Photos configuration
if config.DestinationType != "gdrive" && config.DestinationType != "gphotos" {
RenderErrorPage(c, "Not a Google configuration", "The selected configuration is not set up for Google Drive or Google Photos")
return
}
// Prepare for OAuth
dataDir := os.Getenv("DATA_DIR")
if dataDir == "" {
dataDir = "./data"
}
// Get Rclone Config Path
rcloneConfigPath := h.DB.GetConfigRclonePath(config)
if rcloneConfigPath == "" {
RenderErrorPage(c, "Rclone config not found", "The selected configuration does not have a valid rclone config")
return
}
// Create a temporary config file for authentication
tempConfigDir := filepath.Join(dataDir, "temp")
if err := os.MkdirAll(tempConfigDir, 0755); err != nil {
RenderErrorPage(c, "Failed to create temporary directory", err.Error())
return
}
tempConfigPath := filepath.Join(tempConfigDir, fmt.Sprintf("gdrive_auth_%d.conf", config.ID))
// Store the temporary config path in a cookie
c.SetCookie("gdrive_temp_config", tempConfigPath, 3600, "/", "", false, true)
// Get base URL for redirect URI
baseURL := os.Getenv("BASE_URL")
if baseURL == "" {
// Try to detect the base URL from the request
scheme := "http"
if c.Request.TLS != nil {
scheme = "https"
}
baseURL = fmt.Sprintf("%s://%s", scheme, c.Request.Host)
}
// Define the redirect URI for our callback
redirectURI := fmt.Sprintf("%s/configs/gdrive-callback", baseURL)
// Attempt to get GOOGLE_CLIENT_ID and GOOGLE_CLIENT_SECRET from ENV
clientID := os.Getenv("GOOGLE_CLIENT_ID")
clientSecret := os.Getenv("GOOGLE_CLIENT_SECRET")
if clientID == "" || clientSecret == "" {
// Check if we have client credentials in the existing config file
existingClientID, existingClientSecret := h.DB.GetGDriveCredentialsFromConfig(config)
if existingClientID != "" && existingClientSecret != "" {
// Use credentials from existing config
clientID = existingClientID
clientSecret = existingClientSecret
} else {
// fallback to rclone client ID and secret
clientID = "202264815644.apps.googleusercontent.com"
clientSecret = "X4Z3ca8xfWDb1Voo-F9a7ZxJ"
}
}
// Generate state parameter for security (to prevent CSRF)
state := fmt.Sprintf("gomft_%d_%d", config.ID, time.Now().Unix())
// Store state in cookie for validation during callback
c.SetCookie("gdrive_auth_state", state, 3600, "/", "", false, true)
// Store config ID in cookie for use during callback
c.SetCookie("gdrive_config_id", configIDStr, 3600, "/", "", false, true)
// Determine the appropriate scope based on destination type
var scope string
if config.DestinationType == "gphotos" {
// Read-only access is handled elsewhere in the config; here we need the full auth scope
scope = url.QueryEscape("https://www.googleapis.com/auth/photoslibrary")
} else {
// Default to Google Drive scope
scope = url.QueryEscape("https://www.googleapis.com/auth/drive")
}
// Direct Google OAuth URL with our redirect
authURL := fmt.Sprintf("https://accounts.google.com/o/oauth2/auth?client_id=%s&redirect_uri=%s&scope=%s&response_type=code&access_type=offline&state=%s",
url.QueryEscape(clientID),
url.QueryEscape(redirectURI),
scope,
url.QueryEscape(state))
// Redirect the user to Google's auth page directly
c.Redirect(http.StatusFound, authURL)
}
// HandleGDriveAuthCallback handles the callback from Google OAuth
func (h *MockHandlers) HandleGDriveAuthCallback(c *gin.Context) {
// Get auth code from query parameters
authCode := c.Query("code")
if authCode == "" {
RenderErrorPage(c, "Authentication failed", "No authorization code received from Google")
return
}
// Verify state parameter to prevent CSRF
state := c.Query("state")
storedState, err := c.Cookie("gdrive_auth_state")
if err != nil || state != storedState {
RenderErrorPage(c, "Authentication failed", "Invalid state parameter")
return
}
// Get config ID from cookie
configIDStr, err := c.Cookie("gdrive_config_id")
if err != nil {
RenderErrorPage(c, "Authentication failed", "Unable to retrieve configuration ID")
return
}
configID, err := strconv.ParseUint(configIDStr, 10, 64)
if err != nil {
RenderErrorPage(c, "Invalid configuration ID", err.Error())
return
}
// Get the configuration
config, err := h.DB.GetTransferConfig(uint(configID))
if err != nil {
RenderErrorPage(c, "Failed to get configuration", err.Error())
return
}
// For testing purposes, we'll simulate a successful token exchange
// In a real implementation, we would exchange the auth code for a token
mockToken := `{"access_token":"test_access_token","refresh_token":"test_refresh_token","expiry":"2023-12-31T23:59:59Z"}`
// Update the config with the token
err = h.DB.GenerateRcloneConfigWithToken(config, mockToken)
if err != nil {
RenderErrorPage(c, "Failed to update configuration", err.Error())
return
}
// Redirect to the config edit page
c.Redirect(http.StatusFound, fmt.Sprintf("/configs/edit/%d", config.ID))
}
func setupTestRouter() (*gin.Engine, *MockDB) {
gin.SetMode(gin.TestMode)
router := gin.New()
mockDB := new(MockDB)
handlers := &MockHandlers{
DB: mockDB,
}
router.GET("/configs/gdrive/:id", handlers.HandleGDriveAuth)
router.GET("/configs/gdrive-callback", handlers.HandleGDriveAuthCallback)
return router, mockDB
}
func TestHandleGDriveAuth_GoogleDrive(t *testing.T) {
// Setup
router, mockDB := setupTestRouter()
// Create a test config
testConfig := &db.TransferConfig{
ID: 1,
DestinationType: "gdrive",
}
// Set up mock expectations
mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil)
mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf")
mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret")
// Create test request
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/configs/gdrive/1", nil)
router.ServeHTTP(w, req)
// Assertions
assert.Equal(t, http.StatusFound, w.Code)
// Verify the redirect URL
location := w.Header().Get("Location")
assert.Contains(t, location, "accounts.google.com/o/oauth2/auth")
assert.Contains(t, location, "drive")
assert.Contains(t, location, "test_client_id")
// Verify cookies were set
cookies := w.Result().Cookies()
assert.GreaterOrEqual(t, len(cookies), 3)
// Check if state cookie exists
stateFound := false
for _, cookie := range cookies {
if cookie.Name == "gdrive_auth_state" {
stateFound = true
break
}
}
assert.True(t, stateFound)
}
func TestHandleGDriveAuth_GooglePhotos(t *testing.T) {
// Setup
router, mockDB := setupTestRouter()
// Create a test config
testConfig := &db.TransferConfig{
ID: 2,
DestinationType: "gphotos",
}
// Set up mock expectations
mockDB.On("GetTransferConfig", uint(2)).Return(testConfig, nil)
mockDB.On("GetConfigRclonePath", testConfig).Return("/path/to/rclone.conf")
mockDB.On("GetGDriveCredentialsFromConfig", testConfig).Return("test_client_id", "test_client_secret")
// Create test request
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/configs/gdrive/2", nil)
router.ServeHTTP(w, req)
// Assertions
assert.Equal(t, http.StatusFound, w.Code)
// Verify the redirect URL
location := w.Header().Get("Location")
assert.Contains(t, location, "accounts.google.com/o/oauth2/auth")
assert.Contains(t, location, "photoslibrary")
assert.Contains(t, location, "test_client_id")
// Verify cookies were set
cookies := w.Result().Cookies()
assert.GreaterOrEqual(t, len(cookies), 3)
// Check if state cookie exists
stateFound := false
for _, cookie := range cookies {
if cookie.Name == "gdrive_auth_state" {
stateFound = true
break
}
}
assert.True(t, stateFound)
}
func TestHandleGDriveAuthCallback(t *testing.T) {
// Setup test environment
router, mockDB := setupTestRouter()
// Create a temporary directory for testing
tempDir, err := os.MkdirTemp("", "gdrive-test")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tempDir)
// Create a temporary config file
tempConfigPath := filepath.Join(tempDir, "temp_config.conf")
if err := os.WriteFile(tempConfigPath, []byte("test config"), 0644); err != nil {
t.Fatal(err)
}
// Test state and config ID
testState := "gomft_1_12345"
testConfigID := "1"
// Create a test config
testConfig := &db.TransferConfig{
ID: 1,
DestinationType: "gphotos",
}
// Set up mock expectations
mockDB.On("GetTransferConfig", uint(1)).Return(testConfig, nil)
mockDB.On("GenerateRcloneConfigWithToken", testConfig, mock.Anything).Return(nil)
// Create test request with auth code and state
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/configs/gdrive-callback?code=test_auth_code&state="+testState, nil)
// Add required cookies to the request
req.AddCookie(&http.Cookie{Name: "gdrive_auth_state", Value: testState})
req.AddCookie(&http.Cookie{Name: "gdrive_config_id", Value: testConfigID})
req.AddCookie(&http.Cookie{Name: "gdrive_temp_config", Value: tempConfigPath})
// Send the request
router.ServeHTTP(w, req)
// We expect a redirect on successful auth
assert.Equal(t, http.StatusFound, w.Code)
// Should redirect to the config edit page
location := w.Header().Get("Location")
assert.Contains(t, location, "/configs/edit/1")
}
func TestHandleGDriveAuth_InvalidConfig(t *testing.T) {
// Setup
router, mockDB := setupTestRouter()
// Set up mock expectations for a non-existent config
mockDB.On("GetTransferConfig", uint(999)).Return(nil, fmt.Errorf("config not found"))
// Create test request
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/configs/gdrive/999", nil)
router.ServeHTTP(w, req)
// Assertions - should render error page
assert.Equal(t, http.StatusOK, w.Code)
assert.Contains(t, w.Body.String(), "Configuration not found")
}
func TestHandleGDriveAuth_NonGoogleConfig(t *testing.T) {
// Setup
router, mockDB := setupTestRouter()
// Create a non-Google test config
testConfig := &db.TransferConfig{
ID: 3,
DestinationType: "s3", // Not Google Drive or Photos
}
// Set up mock expectations
mockDB.On("GetTransferConfig", uint(3)).Return(testConfig, nil)
// Create test request
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/configs/gdrive/3", nil)
router.ServeHTTP(w, req)
// Assertions - should render error page
assert.Equal(t, http.StatusOK, w.Code)
assert.Contains(t, w.Body.String(), "Not a Google configuration")
}
// RenderErrorPage renders an error page with the given message
func RenderErrorPage(c *gin.Context, title string, details string) {
// Here we'd typically use a component for error display
// For now, we'll just render a simple HTML error page for testing
errorHTML := fmt.Sprintf("<html><body><h1>Error: %s</h1><p>%s</p></body></html>", title, details)
c.Data(http.StatusOK, "text/html", []byte(errorHTML))
}