From 11f8631929145d35367fede8e679524b55d8b1a5 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 17:30:35 -0700 Subject: [PATCH 1/7] feat: Add support for Backblaze B2 and Wasabi cloud storage - Introduced new source and destination forms for Backblaze B2 and Wasabi, allowing users to configure their cloud storage settings. - Updated the configuration generation logic to handle B2 and Wasabi, including necessary parameters such as access keys, bucket names, and endpoints. - Enhanced the user interface to provide clear instructions and security notes for both storage options, improving user experience and security awareness. --- components/config_form.templ | 49 +++++++- components/providers/common/common.templ | 4 + components/providers/destination/b2.templ | 97 +++++++++++++++ components/providers/destination/wasabi.templ | 112 ++++++++++++++++++ components/providers/source/b2.templ | 97 +++++++++++++++ components/providers/source/wasabi.templ | 112 ++++++++++++++++++ internal/db/transfer_config_store.go | 74 ++++++++++++ internal/rclone_service/rclone_service.go | 30 +++++ 8 files changed, 571 insertions(+), 4 deletions(-) create mode 100644 components/providers/destination/b2.templ create mode 100644 components/providers/destination/wasabi.templ create mode 100644 components/providers/source/b2.templ create mode 100644 components/providers/source/wasabi.templ diff --git a/components/config_form.templ b/components/config_form.templ index ca6b74e..7068c13 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -509,15 +509,23 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { - + + + + + - + @@ -525,7 +533,7 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { - + @@ -533,7 +541,7 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { - + @@ -589,6 +597,14 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @destination.S3DestinationForm() + + + + @@ -626,6 +642,31 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @common.ArchiveOptions() +
+ +
+
+ +
+ +
+

+ Directory where files will be saved + Remote path on the server + Path within bucket. Leave empty for bucket root. + Share directory path + Path on remote server + Folder in Google Drive + Album name in Google Photos +

+
+ diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index ddde8ea..a2df517 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -195,6 +195,8 @@ templ SourceSelection() { + + @@ -219,6 +221,8 @@ templ DestinationSelection() { + + diff --git a/components/providers/destination/b2.templ b/components/providers/destination/b2.templ new file mode 100644 index 0000000..f563e71 --- /dev/null +++ b/components/providers/destination/b2.templ @@ -0,0 +1,97 @@ +package destination + +templ B2DestinationForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Name of your Backblaze B2 bucket (case-sensitive) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Backblaze B2 Account ID +

+
+ +
+ +
+
+ +
+ +
+

+ Your Backblaze B2 Application Key +

+
+ +
+ +
+
+ +
+ +
+

+ Custom endpoint URL (only needed for non-standard regions or private endpoints) +

+
+ +
+ +
+
+ +
+ +
+

+ Path prefix within the bucket (e.g., "backups/"). Leave empty to access the entire bucket. +

+
+ + +
+} \ No newline at end of file diff --git a/components/providers/destination/wasabi.templ b/components/providers/destination/wasabi.templ new file mode 100644 index 0000000..6cabf92 --- /dev/null +++ b/components/providers/destination/wasabi.templ @@ -0,0 +1,112 @@ +package destination + +templ WasabiDestinationForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Wasabi region where your bucket is located (e.g., us-east-1, us-west-1, eu-central-1) +

+
+ +
+ +
+
+ +
+ +
+

+ Name of your Wasabi bucket (case-sensitive) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Wasabi Access Key +

+
+ +
+ +
+
+ +
+ +
+

+ Your Wasabi Secret Key +

+
+ +
+ +
+
+ +
+ +
+

+ Wasabi endpoint URL (usually s3.wasabisys.com or region-specific endpoints) +

+
+ +
+ +
+
+ +
+ +
+

+ Path prefix within the bucket (e.g., "backups/"). Leave empty to access the entire bucket. +

+
+ + +
+} \ No newline at end of file diff --git a/components/providers/source/b2.templ b/components/providers/source/b2.templ new file mode 100644 index 0000000..a75f327 --- /dev/null +++ b/components/providers/source/b2.templ @@ -0,0 +1,97 @@ +package source + +templ B2SourceForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Name of your Backblaze B2 bucket (case-sensitive) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Backblaze B2 Account ID +

+
+ +
+ +
+
+ +
+ +
+

+ Your Backblaze B2 Application Key +

+
+ +
+ +
+
+ +
+ +
+

+ Custom endpoint URL (only needed for non-standard regions or private endpoints) +

+
+ +
+ +
+
+ +
+ +
+

+ Path prefix within the bucket (e.g., "backups/"). Leave empty to access the entire bucket. +

+
+ + +
+} \ No newline at end of file diff --git a/components/providers/source/wasabi.templ b/components/providers/source/wasabi.templ new file mode 100644 index 0000000..84292f1 --- /dev/null +++ b/components/providers/source/wasabi.templ @@ -0,0 +1,112 @@ +package source + +templ WasabiSourceForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Wasabi region where your bucket is located (e.g., us-east-1, us-west-1, eu-central-1) +

+
+ +
+ +
+
+ +
+ +
+

+ Name of your Wasabi bucket (case-sensitive) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Wasabi Access Key +

+
+ +
+ +
+
+ +
+ +
+

+ Your Wasabi Secret Key +

+
+ +
+ +
+
+ +
+ +
+

+ Wasabi endpoint URL (usually s3.wasabisys.com or region-specific endpoints) +

+
+ +
+ +
+
+ +
+ +
+

+ Path prefix within the bucket (e.g., "backups/"). Leave empty to access the entire bucket. +

+
+ + +
+} \ No newline at end of file diff --git a/internal/db/transfer_config_store.go b/internal/db/transfer_config_store.go index 3712782..19834ab 100644 --- a/internal/db/transfer_config_store.go +++ b/internal/db/transfer_config_store.go @@ -130,6 +130,43 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create source config (s3): %v\nOutput: %s", err, output) } + case "wasabi": + args := []string{ + "config", "create", sourceName, "s3", + "provider", "Wasabi", + "env_auth", "false", + "access_key_id", config.SourceAccessKey, + "secret_access_key", config.SourceSecretKey, + "region", config.SourceRegion, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + endpoint := config.SourceEndpoint + if endpoint == "" { + endpoint = "s3.wasabisys.com" + } + args = append(args, "endpoint", endpoint) + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config (wasabi): %v\nOutput: %s", err, output) + } + case "b2": + args := []string{ + "config", "create", sourceName, "b2", + "account", config.SourceAccessKey, // B2 Account ID + "key", config.SourceSecretKey, // B2 Application Key + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + if config.SourceEndpoint != "" { + args = append(args, "endpoint", config.SourceEndpoint) + } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create source config (b2): %v\nOutput: %s", err, output) + } case "minio": args := []string{ "config", "create", sourceName, "s3", @@ -247,6 +284,43 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { if output, err := cmd.CombinedOutput(); err != nil { return fmt.Errorf("failed to create destination config (s3): %v\nOutput: %s", err, output) } + case "wasabi": + args := []string{ + "config", "create", destName, "s3", + "provider", "Wasabi", + "env_auth", "false", + "access_key_id", config.DestAccessKey, + "secret_access_key", config.DestSecretKey, + "region", config.DestRegion, + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + endpoint := config.DestEndpoint + if endpoint == "" { + endpoint = "s3.wasabisys.com" + } + args = append(args, "endpoint", endpoint) + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config (wasabi): %v\nOutput: %s", err, output) + } + case "b2": + args := []string{ + "config", "create", destName, "b2", + "account", config.DestAccessKey, // B2 Account ID + "key", config.DestSecretKey, // B2 Application Key + "--non-interactive", + "--config", configPath, + "--log-level", "ERROR", + } + if config.DestEndpoint != "" { + args = append(args, "endpoint", config.DestEndpoint) + } + cmd := exec.Command(rclonePath, args...) + if output, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("failed to create destination config (b2): %v\nOutput: %s", err, output) + } case "minio": args := []string{ "config", "create", destName, "s3", diff --git a/internal/rclone_service/rclone_service.go b/internal/rclone_service/rclone_service.go index e527a01..002fecb 100644 --- a/internal/rclone_service/rclone_service.go +++ b/internal/rclone_service/rclone_service.go @@ -130,6 +130,36 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta if endpoint != "" { createArgs = append(createArgs, "endpoint", endpoint) } + case "wasabi": + createArgs = append(createArgs, "provider", "Wasabi", "env_auth", "false") + if accessKey != "" { + createArgs = append(createArgs, "access_key_id", accessKey) + } + if secretKey != "" { + createArgs = append(createArgs, "secret_access_key", secretKey) + } + if region != "" { + createArgs = append(createArgs, "region", region) + } + endpointValue := endpoint + if endpointValue == "" { + endpointValue = "s3.wasabisys.com" + } + createArgs = append(createArgs, "endpoint", endpointValue) + case "b2": + createArgs = append(createArgs, "provider", "B2", "env_auth", "false") + if accessKey != "" { + createArgs = append(createArgs, "account", accessKey) + } + if secretKey != "" { + createArgs = append(createArgs, "key", secretKey) + } + if endpoint != "" { + createArgs = append(createArgs, "endpoint", endpoint) + } + if region != "" { + createArgs = append(createArgs, "region", region) + } case "minio": createArgs = append(createArgs, "provider", "Minio", "env_auth", "false") if accessKey != "" { From 417841b3d282f9f12fb1dac289ac196b3173d5c4 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 17:45:10 -0700 Subject: [PATCH 2/7] refactor: Enhance user and group management in entrypoint script - Added checks for existing groups and users before creation to prevent conflicts and improve error handling. - Updated user and group creation logic to use existing identifiers when available, ensuring smoother execution. - Improved ownership setting for application directories and .env file to accommodate potential changes in group names. - Enhanced verification of UID/GID changes with clearer warning messages for better user feedback. --- entrypoint.sh | 75 ++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 19 deletions(-) diff --git a/entrypoint.sh b/entrypoint.sh index bf2f734..9d14d0e 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -26,12 +26,22 @@ if [ -n "${PUID}" ] && [ -n "${PGID}" ]; then deluser ${USERNAME} > /dev/null 2>&1 || true delgroup ${USERNAME} > /dev/null 2>&1 || true - # Add group with the specified GID - echo "Adding group ${USERNAME} with GID ${PGID}" - if ! addgroup -g ${PGID} ${USERNAME}; then - echo "⚠️ Failed to add group ${USERNAME} with GID ${PGID}." - # Exiting because user creation will likely fail - exit 1 + # Check if a group with the target GID already exists + EXISTING_GROUP=$(getent group ${PGID} | cut -d: -f1 || echo "") + + if [ -n "${EXISTING_GROUP}" ]; then + echo "Group with GID ${PGID} already exists as '${EXISTING_GROUP}', will use this group" + # Set USERNAME_GROUP to the existing group name + USERNAME_GROUP="${EXISTING_GROUP}" + else + # Add group with the specified GID + echo "Adding group ${USERNAME} with GID ${PGID}" + if ! addgroup -g ${PGID} ${USERNAME}; then + echo "⚠️ Failed to add group ${USERNAME} with GID ${PGID}." + # Exiting because user creation will likely fail + exit 1 + fi + USERNAME_GROUP="${USERNAME}" fi # Add user with the specified UID and GID @@ -39,16 +49,16 @@ if [ -n "${PUID}" ] && [ -n "${PGID}" ]; then # Use -h /app for home directory (consistent with expectations) # Use -s /bin/sh for shell # Use -D for no password (system user) - echo "Adding user ${USERNAME} with UID ${PUID}" - if ! adduser -u ${PUID} -G ${USERNAME} -h /app -s /bin/sh -D ${USERNAME}; then - echo "⚠️ Failed to add user ${USERNAME} with UID ${PUID} and group ${USERNAME}." + echo "Adding user ${USERNAME} with UID ${PUID} and group ${USERNAME_GROUP}" + if ! adduser -u ${PUID} -G ${USERNAME_GROUP} -h /app -s /bin/sh -D ${USERNAME}; then + echo "⚠️ Failed to add user ${USERNAME} with UID ${PUID} and group ${USERNAME_GROUP}." # Exiting because the application cannot run as the correct user exit 1 fi # Verify the change FINAL_UID=$(id -u ${USERNAME} 2>/dev/null || echo "error") - FINAL_GID=$(getent group ${USERNAME} | cut -d: -f3 2>/dev/null || echo "error") + FINAL_GID=$(id -g ${USERNAME} 2>/dev/null || echo "error") if [ "${FINAL_UID}" = "${PUID}" ] && [ "${FINAL_GID}" = "${PGID}" ]; then echo "✅ Successfully updated UID/GID to ${PUID}:${PGID}" @@ -79,13 +89,40 @@ if [ -n "${PUID}" ] && [ -n "${PGID}" ]; then groupdel ${USERNAME} 2>/dev/null || true fi - # Recreate group and user in the correct order - echo "Creating group ${USERNAME} with GID ${PGID}" - groupadd -g ${PGID} ${USERNAME} 2>/dev/null || groupadd ${USERNAME} 2>/dev/null || true + # Check if a group with the target GID already exists + EXISTING_GROUP=$(getent group ${PGID} | cut -d: -f1 || echo "") - echo "Creating user ${USERNAME} with UID ${PUID}" - useradd -u ${PUID} -g ${USERNAME} -s /bin/sh ${USERNAME} 2>/dev/null || - useradd -g ${USERNAME} -s /bin/sh ${USERNAME} 2>/dev/null || true + if [ -n "${EXISTING_GROUP}" ]; then + echo "Group with GID ${PGID} already exists as '${EXISTING_GROUP}', will use this group" + # Set USERNAME_GROUP to the existing group name + USERNAME_GROUP="${EXISTING_GROUP}" + else + # Recreate group with the specified GID + echo "Creating group ${USERNAME} with GID ${PGID}" + groupadd -g ${PGID} ${USERNAME} 2>/dev/null || groupadd ${USERNAME} 2>/dev/null || true + USERNAME_GROUP="${USERNAME}" + fi + + # Check if a user with the target UID already exists + EXISTING_USER=$(getent passwd ${PUID} | cut -d: -f1 || echo "") + + if [ -n "${EXISTING_USER}" ] && [ "${EXISTING_USER}" != "${USERNAME}" ]; then + echo "⚠️ Warning: User with UID ${PUID} already exists as '${EXISTING_USER}'. Using a different username may cause issues." + fi + + echo "Creating user ${USERNAME} with UID ${PUID} and group ${USERNAME_GROUP}" + useradd -u ${PUID} -g ${USERNAME_GROUP} -s /bin/sh ${USERNAME} 2>/dev/null || + useradd -g ${USERNAME_GROUP} -s /bin/sh ${USERNAME} 2>/dev/null || true + + # Verify the change + FINAL_UID=$(id -u ${USERNAME} 2>/dev/null || echo "error") + FINAL_GID=$(id -g ${USERNAME} 2>/dev/null || echo "error") + + if [ "${FINAL_UID}" = "${PUID}" ] && [ "${FINAL_GID}" = "${PGID}" ]; then + echo "✅ Successfully updated UID/GID to ${PUID}:${PGID}" + else + echo "⚠️ Warning: Verification failed. Target: ${PUID}:${PGID}, Actual: ${FINAL_UID}:${FINAL_GID}" + fi } || { echo "⚠️ Warning: Failed to update UID/GID, continuing with built-in user" } @@ -93,17 +130,17 @@ if [ -n "${PUID}" ] && [ -n "${PGID}" ]; then # Fix ownership of app directories echo "Setting ownership of app directories" - chown -R ${USERNAME}:${USERNAME} /app/data /app/backups || echo "⚠️ Warning: Failed to change ownership" + chown -R ${USERNAME}:${USERNAME_GROUP:-${USERNAME}} /app/data /app/backups || echo "⚠️ Warning: Failed to change ownership" # Ensure .env file exists and has correct permissions if [ -f /app/.env ]; then echo "Found .env file, setting permissions..." - chown ${USERNAME}:${USERNAME} /app/.env || echo "⚠️ Warning: Failed to change .env ownership" + chown ${USERNAME}:${USERNAME_GROUP:-${USERNAME}} /app/.env || echo "⚠️ Warning: Failed to change .env ownership" chmod 644 /app/.env || echo "⚠️ Warning: Failed to change .env permissions" else echo "No .env file found, creating empty one..." touch /app/.env - chown ${USERNAME}:${USERNAME} /app/.env || echo "⚠️ Warning: Failed to change .env ownership" + chown ${USERNAME}:${USERNAME_GROUP:-${USERNAME}} /app/.env || echo "⚠️ Warning: Failed to change .env ownership" chmod 644 /app/.env || echo "⚠️ Warning: Failed to change .env permissions" fi From de97ad6b504831dd7b196ebab8eb320833b84c1d Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 20:24:13 -0700 Subject: [PATCH 3/7] feat: Add support for Hetzner Storage Box integration - Introduced new source and destination forms for Hetzner Storage Box, allowing users to configure their storage settings. - Updated configuration generation logic to handle Hetzner, including necessary parameters such as host, port, username, and authentication type. - Enhanced user interface with clear instructions and security notes for Hetzner, improving user experience and security awareness. - Updated existing logic to accommodate Hetzner in rclone configuration generation for both source and destination types. --- components/config_form.templ | 52 ++++--- components/providers/common/common.templ | 2 + .../providers/destination/hetzner.templ | 128 ++++++++++++++++++ components/providers/source/hetzner.templ | 128 ++++++++++++++++++ internal/db/transfer_config_store.go | 4 +- internal/rclone_service/rclone_service.go | 11 ++ 6 files changed, 294 insertions(+), 31 deletions(-) create mode 100644 components/providers/destination/hetzner.templ create mode 100644 components/providers/source/hetzner.templ diff --git a/components/config_form.templ b/components/config_form.templ index 7068c13..14d2957 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -398,6 +398,8 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { sourcePort = 22; } else if (sourceType === 'ftp') { sourcePort = 21; + } else if (sourceType === 'hetzner') { + sourcePort = 23; } } @@ -406,25 +408,32 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { destPort = 22; } else if (destinationType === 'ftp') { destPort = 21; + } else if (destinationType === 'hetzner') { + destPort = 23; } } // Initialize command requirements updateCommandRequirements(); })" - x-effect="if (sourceType === 'sftp' && (sourcePort === 0 || sourcePort === 21)) { + x-effect="if (sourceType === 'sftp' && (sourcePort === 0 || sourcePort === 21 || sourcePort === 23)) { sourcePort = 22; console.log('Updating source port to 22 for SFTP'); - } else if (sourceType === 'ftp' && (sourcePort === 0 || sourcePort === 22)) { + } else if (sourceType === 'ftp' && (sourcePort === 0 || sourcePort === 22 || sourcePort === 23)) { sourcePort = 21; console.log('Updating source port to 21 for FTP'); - }" - x-effect="if (destinationType === 'sftp' && (destPort === 0 || destPort === 21)) { + } else if (sourceType === 'hetzner' && (sourcePort === 0 || sourcePort === 22 || sourcePort === 21)) { + sourcePort = 23; + console.log('Updating source port to 23 for Hetzner'); + } else if (destinationType === 'sftp' && (destPort === 0 || destPort === 21 || destPort === 23)) { destPort = 22; console.log('Updating destination port to 22 for SFTP'); - } else if (destinationType === 'ftp' && (destPort === 0 || destPort === 22)) { + } else if (destinationType === 'ftp' && (destPort === 0 || destPort === 22 || destPort === 23)) { destPort = 21; console.log('Updating destination port to 21 for FTP'); + } else if (destinationType === 'hetzner' && (destPort === 0 || destPort === 22 || destPort === 21)) { + destPort = 23; + console.log('Updating destination port to 23 for Hetzner'); }" > @@ -545,6 +554,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + @@ -628,6 +641,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { + + @@ -642,30 +659,7 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { @common.ArchiveOptions() -
- -
-
- -
- -
-

- Directory where files will be saved - Remote path on the server - Path within bucket. Leave empty for bucket root. - Share directory path - Path on remote server - Folder in Google Drive - Album name in Google Photos -

-
+ diff --git a/components/providers/common/common.templ b/components/providers/common/common.templ index a2df517..5323293 100644 --- a/components/providers/common/common.templ +++ b/components/providers/common/common.templ @@ -203,6 +203,7 @@ templ SourceSelection() { + @@ -229,6 +230,7 @@ templ DestinationSelection() { + diff --git a/components/providers/destination/hetzner.templ b/components/providers/destination/hetzner.templ new file mode 100644 index 0000000..fd26b9d --- /dev/null +++ b/components/providers/destination/hetzner.templ @@ -0,0 +1,128 @@ +package destination + +templ HetznerDestinationForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box hostname (e.g., uXXXXXX.your-storagebox.de) +

+
+ +
+ +
+
+ +
+ +
+

+ Connection port (default: 23) +

+
+ +
+ +
+
+ + +
+
+ + +
+
+
+ +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box username (typically matches your Storage Box number) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box password +

+
+ +
+ +
+
+ +
+ +
+

+ Path to your SSH private key file (must be readable by the application) +

+
+ +
+ +
+
+ +
+ +
+

+ Path on your Storage Box (e.g., /backups, /path/to/files) +

+
+ + +
+} \ No newline at end of file diff --git a/components/providers/source/hetzner.templ b/components/providers/source/hetzner.templ new file mode 100644 index 0000000..554229a --- /dev/null +++ b/components/providers/source/hetzner.templ @@ -0,0 +1,128 @@ +package source + +templ HetznerSourceForm() { +
+ + +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box hostname (e.g., uXXXXXX.your-storagebox.de) +

+
+ +
+ +
+
+ +
+ +
+

+ Connection port (default: 23) +

+
+ +
+ +
+
+ + +
+
+ + +
+
+
+ +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box username (typically matches your Storage Box number) +

+
+ +
+ +
+
+ +
+ +
+

+ Your Hetzner Storage Box password +

+
+ +
+ +
+
+ +
+ +
+

+ Path to your SSH private key file (must be readable by the application) +

+
+ +
+ +
+
+ +
+ +
+

+ Path on your Storage Box (e.g., /backups, /path/to/files) +

+
+ + +
+} \ No newline at end of file diff --git a/internal/db/transfer_config_store.go b/internal/db/transfer_config_store.go index 19834ab..08ee310 100644 --- a/internal/db/transfer_config_store.go +++ b/internal/db/transfer_config_store.go @@ -91,7 +91,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { sourceName := fmt.Sprintf("source_%d", config.ID) // Generate rclone config using rclone CLI for source switch config.SourceType { - case "sftp": + case "sftp", "hetzner": args := []string{ "config", "create", sourceName, "sftp", "host", config.SourceHost, @@ -245,7 +245,7 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error { destName := fmt.Sprintf("dest_%d", config.ID) // Generate rclone config using rclone CLI for destination switch config.DestinationType { - case "sftp": + case "sftp", "hetzner": args := []string{ "config", "create", destName, "sftp", "host", config.DestHost, diff --git a/internal/rclone_service/rclone_service.go b/internal/rclone_service/rclone_service.go index 002fecb..7796936 100644 --- a/internal/rclone_service/rclone_service.go +++ b/internal/rclone_service/rclone_service.go @@ -116,6 +116,17 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta if keyFile != "" { createArgs = append(createArgs, "key_file", keyFile) } + case "hetzner": + createArgs = append(createArgs, "host", host, "user", user, "pass", pass, "key_file", keyFile) + if port != 0 { + createArgs = append(createArgs, "port", fmt.Sprintf("%d", port)) + } + if pass != "" { + createArgs = append(createArgs, "pass", pass) + } + if keyFile != "" { + createArgs = append(createArgs, "key_file", keyFile) + } case "s3": createArgs = append(createArgs, "provider", "AWS", "env_auth", "false") if accessKey != "" { From bcd09af2d60ac2a9f87cbdfe79814348af1dd7ef Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 20:33:32 -0700 Subject: [PATCH 4/7] fix: Consolidate Hetzner support under SFTP provider - Updated the rclone connection logic to treat Hetzner as an SFTP provider, simplifying the configuration process. - Removed redundant case handling for Hetzner, streamlining the code and enhancing maintainability. --- internal/rclone_service/rclone_service.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/internal/rclone_service/rclone_service.go b/internal/rclone_service/rclone_service.go index 7796936..753a071 100644 --- a/internal/rclone_service/rclone_service.go +++ b/internal/rclone_service/rclone_service.go @@ -90,6 +90,10 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta rclonePath = "rclone" } + if provider == "hetzner" { + provider = "sftp" + } + createArgs := []string{ "config", "create", remoteName, provider, "--config", tempConfigPath, @@ -105,7 +109,7 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta var createCmd *exec.Cmd switch provider { - case "sftp": + case "sftp", "hetzner": createArgs = append(createArgs, "host", host, "user", user) if port != 0 { createArgs = append(createArgs, "port", fmt.Sprintf("%d", port)) @@ -116,17 +120,6 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta if keyFile != "" { createArgs = append(createArgs, "key_file", keyFile) } - case "hetzner": - createArgs = append(createArgs, "host", host, "user", user, "pass", pass, "key_file", keyFile) - if port != 0 { - createArgs = append(createArgs, "port", fmt.Sprintf("%d", port)) - } - if pass != "" { - createArgs = append(createArgs, "pass", pass) - } - if keyFile != "" { - createArgs = append(createArgs, "key_file", keyFile) - } case "s3": createArgs = append(createArgs, "provider", "AWS", "env_auth", "false") if accessKey != "" { From 78fe589fcfde8c9e3d903dadc6c4b3b4cdc29bcd Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 21:00:41 -0700 Subject: [PATCH 5/7] refactor: Remove recovery migration files for transfer_configs, notification_services, and auth_providers - Deleted migration files that handled recovery for inconsistent states in transfer_configs, notification_services, and auth_providers tables. - Updated migration management to reflect the removal of these recovery steps, ensuring cleaner migration logic. --- ...gs_rename.go => 012a_recover_transfer_configs_rename.go} | 6 +++--- ...name.go => 012b_recover_notification_services_rename.go} | 6 +++--- ...ders_rename.go => 012c_recover_auth_providers_rename.go} | 6 +++--- internal/db/migrations/migrations.go | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) rename internal/db/migrations/{011a_recover_transfer_configs_rename.go => 012a_recover_transfer_configs_rename.go} (91%) rename internal/db/migrations/{011b_recover_notification_services_rename.go => 012b_recover_notification_services_rename.go} (90%) rename internal/db/migrations/{011c_recover_auth_providers_rename.go => 012c_recover_auth_providers_rename.go} (90%) diff --git a/internal/db/migrations/011a_recover_transfer_configs_rename.go b/internal/db/migrations/012a_recover_transfer_configs_rename.go similarity index 91% rename from internal/db/migrations/011a_recover_transfer_configs_rename.go rename to internal/db/migrations/012a_recover_transfer_configs_rename.go index cd6dcf3..8cabb16 100644 --- a/internal/db/migrations/011a_recover_transfer_configs_rename.go +++ b/internal/db/migrations/012a_recover_transfer_configs_rename.go @@ -12,9 +12,9 @@ import ( // table might have been left renamed as _transfer_configs_old. func RecoverTransferConfigsRename() *gormigrate.Migration { return &gormigrate.Migration{ - ID: "011a_recover_transfer_configs_rename", + ID: "012a_recover_transfer_configs_rename", Migrate: func(tx *gorm.DB) error { - fmt.Println("Running migration 011a: Checking for transfer_configs rename recovery...") + fmt.Println("Running migration 012a: Checking for transfer_configs rename recovery...") var oldTableExists int tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_transfer_configs_old'").Scan(&oldTableExists) @@ -40,7 +40,7 @@ func RecoverTransferConfigsRename() *gormigrate.Migration { }, Rollback: func(tx *gorm.DB) error { // Rollback doesn't make sense for a recovery step. - fmt.Println("Rollback for migration 011a_recover_transfer_configs_rename is not applicable.") + fmt.Println("Rollback for migration 012a_recover_transfer_configs_rename is not applicable.") return nil }, } diff --git a/internal/db/migrations/011b_recover_notification_services_rename.go b/internal/db/migrations/012b_recover_notification_services_rename.go similarity index 90% rename from internal/db/migrations/011b_recover_notification_services_rename.go rename to internal/db/migrations/012b_recover_notification_services_rename.go index 5966725..15dfdeb 100644 --- a/internal/db/migrations/011b_recover_notification_services_rename.go +++ b/internal/db/migrations/012b_recover_notification_services_rename.go @@ -12,9 +12,9 @@ import ( // table might have been left renamed as _notification_services_old. func RecoverNotificationServicesRename() *gormigrate.Migration { return &gormigrate.Migration{ - ID: "011b_recover_notification_services_rename", + ID: "012b_recover_notification_services_rename", Migrate: func(tx *gorm.DB) error { - fmt.Println("Running migration 011b: Checking for notification_services rename recovery...") + fmt.Println("Running migration 012b: Checking for notification_services rename recovery...") var oldTableExists int tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_notification_services_old'").Scan(&oldTableExists) @@ -38,7 +38,7 @@ func RecoverNotificationServicesRename() *gormigrate.Migration { }, Rollback: func(tx *gorm.DB) error { // Rollback doesn't make sense for a recovery step. - fmt.Println("Rollback for migration 011b_recover_notification_services_rename is not applicable.") + fmt.Println("Rollback for migration 012b_recover_notification_services_rename is not applicable.") return nil }, } diff --git a/internal/db/migrations/011c_recover_auth_providers_rename.go b/internal/db/migrations/012c_recover_auth_providers_rename.go similarity index 90% rename from internal/db/migrations/011c_recover_auth_providers_rename.go rename to internal/db/migrations/012c_recover_auth_providers_rename.go index 510ba97..071fb8f 100644 --- a/internal/db/migrations/011c_recover_auth_providers_rename.go +++ b/internal/db/migrations/012c_recover_auth_providers_rename.go @@ -12,9 +12,9 @@ import ( // table might have been left renamed as _auth_providers_old. func RecoverAuthProvidersRename() *gormigrate.Migration { return &gormigrate.Migration{ - ID: "011c_recover_auth_providers_rename", + ID: "012c_recover_auth_providers_rename", Migrate: func(tx *gorm.DB) error { - fmt.Println("Running migration 011c: Checking for auth_providers rename recovery...") + fmt.Println("Running migration 012c: Checking for auth_providers rename recovery...") var oldTableExists int tx.Raw("SELECT count(*) FROM sqlite_master WHERE type='table' AND name='_auth_providers_old'").Scan(&oldTableExists) @@ -38,7 +38,7 @@ func RecoverAuthProvidersRename() *gormigrate.Migration { }, Rollback: func(tx *gorm.DB) error { // Rollback doesn't make sense for a recovery step. - fmt.Println("Rollback for migration 011c_recover_auth_providers_rename is not applicable.") + fmt.Println("Rollback for migration 012c_recover_auth_providers_rename is not applicable.") return nil }, } diff --git a/internal/db/migrations/migrations.go b/internal/db/migrations/migrations.go index 5367941..4df11e8 100644 --- a/internal/db/migrations/migrations.go +++ b/internal/db/migrations/migrations.go @@ -22,10 +22,10 @@ func GetMigrations(db *gorm.DB) *gormigrate.Gormigrate { AddRcloneTables(), // 009 AddRcloneCommandToConfig(), // 010 AddAuthProviders(), // 011 - RecoverTransferConfigsRename(), // 011a - RecoverNotificationServicesRename(), // 011b - RecoverAuthProvidersRename(), // 011c AlterBooleanDefaults(), // 012 + RecoverTransferConfigsRename(), // 012a + RecoverNotificationServicesRename(), // 012b + RecoverAuthProvidersRename(), // 012c CleanupInvalidBooleans(), // 013 ) From 87bdfc5e9116f0c849347310142b6708b8127ed7 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 21:05:45 -0700 Subject: [PATCH 6/7] update readme.md with new providers --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 94eb62e..15625bc 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,9 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging - SFTP - FTP - SMB/CIFS shares + - Hetzner Storage Box + - Backblaze B2 + - Wasabi - Local filesystem - And more via rclone - **Webhook Notifications**: Receive real-time notifications of job events: @@ -525,6 +528,10 @@ The following fields have been added to the `users` table: - Local filesystem - Amazon S3 - MinIO (S3-compatible storage) + - NextCloud + - Backblaze B2 + - Wasabi + - Hetzner Storage Box - SFTP - FTP - SMB/CIFS shares From e2ab2eb6034d5aa02449dc6bc3d4abfab077e4f8 Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Tue, 8 Apr 2025 21:35:19 -0700 Subject: [PATCH 7/7] refactor: Update rclone connection logic for provider handling - Consolidated provider handling for Hetzner and S3-compatible providers (Minio, Wasabi) to improve clarity and maintainability. - Adjusted configuration arguments to use the primary provider name consistently across different cases. - Enhanced endpoint handling for Minio and B2 providers, ensuring correct parameter usage in rclone configuration generation. --- internal/rclone_service/rclone_service.go | 42 +++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/internal/rclone_service/rclone_service.go b/internal/rclone_service/rclone_service.go index 753a071..ed956ea 100644 --- a/internal/rclone_service/rclone_service.go +++ b/internal/rclone_service/rclone_service.go @@ -90,12 +90,20 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta rclonePath = "rclone" } + primaryProvider := provider + + // Convert hetzner to sftp if provider == "hetzner" { - provider = "sftp" + primaryProvider = "sftp" + } + + // S3 compatible providers + if provider == "minio" || provider == "wasabi" { + primaryProvider = "s3" } createArgs := []string{ - "config", "create", remoteName, provider, + "config", "create", remoteName, primaryProvider, "--config", tempConfigPath, "--non-interactive", "--log-level", "DEBUG", @@ -145,25 +153,9 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta if region != "" { createArgs = append(createArgs, "region", region) } - endpointValue := endpoint - if endpointValue == "" { - endpointValue = "s3.wasabisys.com" - } - createArgs = append(createArgs, "endpoint", endpointValue) - case "b2": - createArgs = append(createArgs, "provider", "B2", "env_auth", "false") - if accessKey != "" { - createArgs = append(createArgs, "account", accessKey) - } - if secretKey != "" { - createArgs = append(createArgs, "key", secretKey) - } if endpoint != "" { createArgs = append(createArgs, "endpoint", endpoint) } - if region != "" { - createArgs = append(createArgs, "region", region) - } case "minio": createArgs = append(createArgs, "provider", "Minio", "env_auth", "false") if accessKey != "" { @@ -172,6 +164,20 @@ func TestRcloneConnection(config db.TransferConfig, providerType string, dbInsta if secretKey != "" { createArgs = append(createArgs, "secret_access_key", secretKey) } + if region != "" { + createArgs = append(createArgs, "region", region) + } + if endpoint != "" { + createArgs = append(createArgs, "endpoint", endpoint) + } + case "b2": + createArgs = append(createArgs, "provider", "B2", "env_auth", "false") + if accessKey != "" { + createArgs = append(createArgs, "account", accessKey) + } + if secretKey != "" { + createArgs = append(createArgs, "key", secretKey) + } if endpoint != "" { createArgs = append(createArgs, "endpoint", endpoint) }