Files
GoMFT/internal/db/transfer_config_store.go
T
StarFleetCPTN de97ad6b50 feat: Add support for Hetzner Storage Box integration
- Introduced new source and destination forms for Hetzner Storage Box, allowing users to configure their storage settings.
- Updated configuration generation logic to handle Hetzner, including necessary parameters such as host, port, username, and authentication type.
- Enhanced user interface with clear instructions and security notes for Hetzner, improving user experience and security awareness.
- Updated existing logic to accommodate Hetzner in rclone configuration generation for both source and destination types.
2025-04-08 20:24:13 -07:00

663 lines
22 KiB
Go

package db
import (
"fmt"
"net/url"
"os"
"os/exec"
"path/filepath"
"regexp"
"strconv"
"strings"
)
// --- TransferConfig Store Methods ---
// CreateTransferConfig creates a new transfer config record
func (db *DB) CreateTransferConfig(config *TransferConfig) error {
return db.Create(config).Error
}
// GetTransferConfigs retrieves all transfer configs for a user
func (db *DB) GetTransferConfigs(userID uint) ([]TransferConfig, error) {
var configs []TransferConfig
err := db.Where("created_by = ?", userID).Find(&configs).Error
return configs, err
}
// GetTransferConfig retrieves a single transfer config by ID
func (db *DB) GetTransferConfig(id uint) (*TransferConfig, error) {
var config TransferConfig
err := db.First(&config, id).Error
if err != nil {
return nil, err
}
return &config, nil
}
// UpdateTransferConfig updates an existing transfer config record
func (db *DB) UpdateTransferConfig(config *TransferConfig) error {
return db.Save(config).Error
}
// DeleteTransferConfig deletes a transfer config record after checking dependencies
func (db *DB) DeleteTransferConfig(id uint) error {
// First check if any jobs are using this config
var count int64
// Need to check both ConfigID and ConfigIDs list
// This check might need refinement depending on how ConfigIDs is used reliably
if err := db.Model(&Job{}).Where("config_id = ? OR config_ids LIKE ?", id, "%"+strconv.FormatUint(uint64(id), 10)+"%").Count(&count).Error; err != nil {
return fmt.Errorf("failed to check for dependent jobs: %v", err)
}
if count > 0 {
return fmt.Errorf("cannot delete config: %d jobs are using this configuration", count)
}
// Delete the config
return db.Delete(&TransferConfig{}, id).Error
}
// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
// Get data directory from environment or use default
dataDir := os.Getenv("DATA_DIR")
if dataDir == "" {
dataDir = "./data"
}
// Store configs in the data directory
return filepath.Join(dataDir, "configs", fmt.Sprintf("config_%d.conf", config.ID))
}
// GenerateRcloneConfig generates the rclone config file content based on TransferConfig
// This function now primarily focuses on generating the content string or calling rclone config create
func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
configPath := db.GetConfigRclonePath(config)
// Get the directory part of the path
configDir := filepath.Dir(configPath)
// Ensure configs directory exists
if err := os.MkdirAll(configDir, 0755); err != nil {
return fmt.Errorf("failed to create configs directory: %v", err)
}
// Get the rclone path from the environment variable or use the default path
rclonePath := os.Getenv("RCLONE_PATH")
if rclonePath == "" {
rclonePath = "rclone"
}
sourceName := fmt.Sprintf("source_%d", config.ID)
// Generate rclone config using rclone CLI for source
switch config.SourceType {
case "sftp", "hetzner":
args := []string{
"config", "create", sourceName, "sftp",
"host", config.SourceHost,
"user", config.SourceUser,
"port", fmt.Sprintf("%d", config.SourcePort),
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourcePassword != "" {
args = append(args, "pass", config.SourcePassword)
}
if config.SourceKeyFile != "" {
args = append(args, "key_file", config.SourceKeyFile)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config (sftp): %v\nOutput: %s", err, output)
}
case "s3":
args := []string{
"config", "create", sourceName, "s3",
"provider", "AWS", // Assuming AWS provider, adjust if needed
"env_auth", "false",
"access_key_id", config.SourceAccessKey,
"secret_access_key", config.SourceSecretKey,
"region", config.SourceRegion,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config (s3): %v\nOutput: %s", err, output)
}
case "wasabi":
args := []string{
"config", "create", sourceName, "s3",
"provider", "Wasabi",
"env_auth", "false",
"access_key_id", config.SourceAccessKey,
"secret_access_key", config.SourceSecretKey,
"region", config.SourceRegion,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
endpoint := config.SourceEndpoint
if endpoint == "" {
endpoint = "s3.wasabisys.com"
}
args = append(args, "endpoint", endpoint)
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config (wasabi): %v\nOutput: %s", err, output)
}
case "b2":
args := []string{
"config", "create", sourceName, "b2",
"account", config.SourceAccessKey, // B2 Account ID
"key", config.SourceSecretKey, // B2 Application Key
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.SourceEndpoint != "" {
args = append(args, "endpoint", config.SourceEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config (b2): %v\nOutput: %s", err, output)
}
case "minio":
args := []string{
"config", "create", sourceName, "s3",
"provider", "Minio",
"env_auth", "false",
"access_key_id", config.SourceAccessKey,
"secret_access_key", config.SourceSecretKey,
"endpoint", config.SourceEndpoint,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
// Add region if specified
if config.SourceRegion != "" {
args = append(args, "region", config.SourceRegion)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create source config (minio): %v\nOutput: %s", err, output)
}
case "webdav", "nextcloud": // Handle both webdav and nextcloud similarly
// Construct the WebDAV URL
// Parse the provided source URL, assuming it includes the scheme
inputURL := config.SourceHost
parsedURL, err := url.Parse(inputURL)
if err != nil {
return fmt.Errorf("failed to parse source URL '%s': %v", inputURL, err)
}
// Validate that both scheme and host are present
if parsedURL.Scheme == "" || parsedURL.Host == "" {
return fmt.Errorf("invalid source URL '%s': must include scheme (http/https) and host", inputURL)
}
// Use the scheme and host from the parsed URL
webdavURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
// Determine vendor based on type
vendor := "other" // Default vendor
if config.SourceType == "nextcloud" {
vendor = "nextcloud"
// Construct the full Nextcloud path using the parsed base URL
webdavURL = fmt.Sprintf("%s/remote.php/dav/files/%s/", webdavURL, config.SourceUser)
}
args := []string{
"config", "create", sourceName, "webdav",
"url", webdavURL,
"vendor", vendor,
"user", config.SourceUser,
"pass", config.SourcePassword, // rclone obscures this
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
errorMsg := fmt.Sprintf("failed to create source config (%s): %v", config.SourceType, err)
// Check if output contains useful info, especially for auth errors
if len(output) > 0 {
errorMsg += fmt.Sprintf("\nOutput: %s", output)
}
return fmt.Errorf(errorMsg)
}
case "local":
// For local source, ensure the section exists but might not need specific rclone config create
content := fmt.Sprintf("[%s]\ntype = local\n\n", sourceName)
if err := os.WriteFile(configPath, []byte(content), 0600); err != nil {
return fmt.Errorf("failed to write source config (local): %v", err)
}
default:
// Handle unknown or unsupported source types if necessary
return fmt.Errorf("unsupported source type for rclone config generation: %s", config.SourceType)
}
destName := fmt.Sprintf("dest_%d", config.ID)
// Generate rclone config using rclone CLI for destination
switch config.DestinationType {
case "sftp", "hetzner":
args := []string{
"config", "create", destName, "sftp",
"host", config.DestHost,
"user", config.DestUser,
"port", fmt.Sprintf("%d", config.DestPort),
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestPassword != "" {
args = append(args, "pass", config.DestPassword)
}
if config.DestKeyFile != "" {
args = append(args, "key_file", config.DestKeyFile)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config (sftp): %v\nOutput: %s", err, output)
}
case "s3":
args := []string{
"config", "create", destName, "s3",
"provider", "AWS", // Assuming AWS provider
"env_auth", "false",
"access_key_id", config.DestAccessKey,
"secret_access_key", config.DestSecretKey,
"region", config.DestRegion,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestEndpoint != "" {
args = append(args, "endpoint", config.DestEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config (s3): %v\nOutput: %s", err, output)
}
case "wasabi":
args := []string{
"config", "create", destName, "s3",
"provider", "Wasabi",
"env_auth", "false",
"access_key_id", config.DestAccessKey,
"secret_access_key", config.DestSecretKey,
"region", config.DestRegion,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
endpoint := config.DestEndpoint
if endpoint == "" {
endpoint = "s3.wasabisys.com"
}
args = append(args, "endpoint", endpoint)
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config (wasabi): %v\nOutput: %s", err, output)
}
case "b2":
args := []string{
"config", "create", destName, "b2",
"account", config.DestAccessKey, // B2 Account ID
"key", config.DestSecretKey, // B2 Application Key
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
if config.DestEndpoint != "" {
args = append(args, "endpoint", config.DestEndpoint)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config (b2): %v\nOutput: %s", err, output)
}
case "minio":
args := []string{
"config", "create", destName, "s3",
"provider", "Minio",
"env_auth", "false",
"access_key_id", config.DestAccessKey,
"secret_access_key", config.DestSecretKey,
"endpoint", config.DestEndpoint,
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
// Add region if specified
if config.DestRegion != "" {
args = append(args, "region", config.DestRegion)
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
return fmt.Errorf("failed to create destination config (minio): %v\nOutput: %s", err, output)
}
case "webdav", "nextcloud": // Combined case for WebDAV and Nextcloud
// Parse and reconstruct the WebDAV URL robustly
// Parse the provided destination URL, assuming it includes the scheme
inputURL := config.DestHost
parsedURL, err := url.Parse(inputURL)
if err != nil {
return fmt.Errorf("failed to parse destination URL '%s': %v", inputURL, err)
}
// Validate that both scheme and host are present
if parsedURL.Scheme == "" || parsedURL.Host == "" {
return fmt.Errorf("invalid destination URL '%s': must include scheme (http/https) and host", inputURL)
}
// Use the scheme and host from the parsed URL
webdavURL := fmt.Sprintf("%s://%s", parsedURL.Scheme, parsedURL.Host)
// Determine vendor based on type
vendor := "other" // Default vendor
if config.DestinationType == "nextcloud" {
vendor = "nextcloud"
webdavURL = fmt.Sprintf("%s/remote.php/dav/files/%s/", webdavURL, config.DestUser) // Corrected variable
}
args := []string{
"config", "create", destName, "webdav",
"url", webdavURL, // Use the parsed and reconstructed URL
"vendor", vendor,
"user", config.DestUser,
"pass", config.DestPassword, // rclone obscures this
"--non-interactive",
"--config", configPath,
"--log-level", "ERROR",
}
cmd := exec.Command(rclonePath, args...)
if output, err := cmd.CombinedOutput(); err != nil {
errorMsg := fmt.Sprintf("failed to create destination config (%s): %v", config.DestinationType, err)
if len(output) > 0 {
errorMsg += fmt.Sprintf("\nOutput: %s", output)
}
return fmt.Errorf(errorMsg)
}
case "local":
// Append local config section
content := fmt.Sprintf("\n[%s]\ntype = local\n", destName)
f, err := os.OpenFile(configPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return fmt.Errorf("failed to open config file for appending (local dest): %v", err)
}
defer f.Close()
if _, err := f.WriteString(content); err != nil {
return fmt.Errorf("failed to write destination config (local): %v", err)
}
default:
// Handle unknown or unsupported destination types if necessary
return fmt.Errorf("unsupported destination type for rclone config generation: %s", config.DestinationType)
}
return nil
}
// StoreGoogleDriveToken stores the Google Drive auth token for a config
func (db *DB) StoreGoogleDriveToken(configIDStr string, token string) error {
configID, err := strconv.ParseUint(configIDStr, 10, 64)
if err != nil {
return fmt.Errorf("invalid config ID: %v", err)
}
config, err := db.GetTransferConfig(uint(configID))
if err != nil {
return fmt.Errorf("failed to get config: %v", err)
}
authenticated := true
config.GoogleDriveAuthenticated = &authenticated
if err := db.UpdateTransferConfig(config); err != nil {
return fmt.Errorf("failed to update config: %v", err)
}
configPath := db.GetConfigRclonePath(config)
existingConfig := ""
if _, err := os.Stat(configPath); err == nil {
data, err := os.ReadFile(configPath)
if err != nil {
return fmt.Errorf("failed to read existing config: %v", err)
}
existingConfig = string(data)
}
configDir := filepath.Dir(configPath)
if err := os.MkdirAll(configDir, 0755); err != nil {
return fmt.Errorf("failed to create config directory: %v", err)
}
destName := fmt.Sprintf("dest_%d", config.ID)
newConfig := fmt.Sprintf("[%s]\ntype = drive\ntoken = %s\n", destName, token)
if config.DestClientID != "" && config.DestClientSecret != "" {
newConfig += fmt.Sprintf("client_id = %s\nclient_secret = %s\n", config.DestClientID, config.DestClientSecret)
}
if config.DestDriveID != "" {
newConfig += fmt.Sprintf("root_folder_id = %s\n", config.DestDriveID)
}
if config.DestTeamDrive != "" {
newConfig += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive)
}
var content string
sectionHeader := fmt.Sprintf("[%s]", destName)
if strings.Contains(existingConfig, sectionHeader) {
parts := strings.SplitN(existingConfig, sectionHeader, 2)
nextSectionIdx := strings.Index(parts[1], "[")
if nextSectionIdx != -1 {
content = parts[0] + newConfig + parts[1][nextSectionIdx:]
} else {
content = parts[0] + newConfig
}
} else {
content = existingConfig + "\n" + newConfig
}
if err := os.WriteFile(configPath, []byte(content), 0600); err != nil {
return fmt.Errorf("failed to write config: %v", err)
}
return nil
}
// GenerateRcloneConfigWithToken generates a rclone config file for a transfer config with a provided token
// Note: This seems partially redundant with StoreGoogleDriveToken and GenerateRcloneConfig. Consolidate if possible.
func (db *DB) GenerateRcloneConfigWithToken(config *TransferConfig, token string) error {
configPath := db.GetConfigRclonePath(config)
if configPath == "" {
return fmt.Errorf("failed to get config path")
}
token = strings.TrimSpace(token)
token = strings.ReplaceAll(token, "\n", "")
token = strings.ReplaceAll(token, "\r", "")
var configType, section, clientID, clientSecret string
var readOnly, includeArchived *bool
var startYear int
// Determine if source or destination needs token update
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
configType = config.DestinationType
section = "dest"
clientID = config.DestClientID
clientSecret = config.DestClientSecret
readOnly = config.DestReadOnly
startYear = config.DestStartYear
includeArchived = config.DestIncludeArchived
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
configType = config.SourceType
section = "source"
clientID = config.SourceClientID
clientSecret = config.SourceClientSecret
readOnly = config.SourceReadOnly
startYear = config.SourceStartYear
includeArchived = config.SourceIncludeArchived
} else {
return fmt.Errorf("config is not for Google Drive or Google Photos")
}
contentBytes, err := os.ReadFile(configPath)
if err != nil && !os.IsNotExist(err) { // Allow file not existing yet
return fmt.Errorf("failed to read config file: %v", err)
}
content := string(contentBytes)
var sectionContent string
sectionHeader := fmt.Sprintf("[%s_%d]", section, config.ID)
if configType == "gdrive" {
sectionContent = sectionHeader + "\ntype = drive\n"
if clientID != "" {
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
}
if clientSecret != "" {
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
}
sectionContent += fmt.Sprintf("token = %s\n", token)
if section == "source" && config.SourceTeamDrive != "" {
sectionContent += fmt.Sprintf("team_drive = %s\n", config.SourceTeamDrive)
}
if section == "dest" && config.DestTeamDrive != "" {
sectionContent += fmt.Sprintf("team_drive = %s\n", config.DestTeamDrive)
}
if section == "dest" && config.DestDriveID != "" {
sectionContent += fmt.Sprintf("root_folder_id = %s\n", config.DestDriveID)
} // Use DestDriveID for root_folder_id
} else if configType == "gphotos" {
sectionContent = sectionHeader + "\ntype = google photos\n"
if clientID != "" {
sectionContent += fmt.Sprintf("client_id = %s\n", clientID)
}
if clientSecret != "" {
sectionContent += fmt.Sprintf("client_secret = %s\n", clientSecret)
}
sectionContent += fmt.Sprintf("token = %s\n", token)
if readOnly != nil && *readOnly {
sectionContent += "read_only = true\n"
}
if startYear > 0 {
sectionContent += fmt.Sprintf("start_year = %d\n", startYear)
}
if includeArchived != nil && *includeArchived {
sectionContent += "include_archived = true\n"
}
}
// Replace or append logic
sectionPattern := regexp.MustCompile(fmt.Sprintf(`(?m)^%s[^\[]*`, regexp.QuoteMeta(sectionHeader))) // Match section start to next section or EOF
if sectionPattern.MatchString(content) {
content = sectionPattern.ReplaceAllString(content, sectionContent)
} else {
if content != "" && !strings.HasSuffix(content, "\n\n") { // Ensure separation
if !strings.HasSuffix(content, "\n") {
content += "\n"
}
content += "\n"
}
content += sectionContent
}
// Ensure directory exists
configDir := filepath.Dir(configPath)
if err := os.MkdirAll(configDir, 0755); err != nil {
return fmt.Errorf("failed to create config directory: %v", err)
}
// Write the updated config file
if err := os.WriteFile(configPath, []byte(content), 0600); err != nil { // Use 0600 for sensitive files
return fmt.Errorf("failed to write updated config file: %v", err)
}
// Update the authentication status in DB
authenticated := true
if config.DestinationType == "gdrive" || config.DestinationType == "gphotos" {
config.SetGoogleAuthenticated(authenticated)
} else if config.SourceType == "gdrive" || config.SourceType == "gphotos" {
config.SetGoogleAuthenticated(authenticated)
}
// Persist the change (assuming UpdateTransferConfig saves the whole object)
if err := db.UpdateTransferConfig(config); err != nil {
return fmt.Errorf("failed to update config authentication status: %v", err)
}
return nil
}
// GetGDriveCredentialsFromConfig extracts Google Drive client ID and secret from an existing rclone config file
func (db *DB) GetGDriveCredentialsFromConfig(config *TransferConfig) (string, string) {
configPath := db.GetConfigRclonePath(config)
if configPath == "" {
return "", ""
}
if _, err := os.Stat(configPath); os.IsNotExist(err) {
return "", ""
}
content, err := os.ReadFile(configPath)
if err != nil {
return "", ""
}
lines := strings.Split(string(content), "\n")
sourceSectionName := fmt.Sprintf("[source_%d]", config.ID)
destSectionName := fmt.Sprintf("[dest_%d]", config.ID)
var inSourceSection, inDestSection bool
var sourceClientID, sourceClientSecret, destClientID, destClientSecret string
for _, line := range lines {
line = strings.TrimSpace(line)
if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") {
inSourceSection = line == sourceSectionName
inDestSection = line == destSectionName
continue
}
if inSourceSection {
if strings.HasPrefix(line, "client_id") {
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 {
sourceClientID = strings.TrimSpace(parts[1])
}
} else if strings.HasPrefix(line, "client_secret") {
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 {
sourceClientSecret = strings.TrimSpace(parts[1])
}
}
}
if inDestSection {
if strings.HasPrefix(line, "client_id") {
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 {
destClientID = strings.TrimSpace(parts[1])
}
} else if strings.HasPrefix(line, "client_secret") {
parts := strings.SplitN(line, "=", 2)
if len(parts) == 2 {
destClientSecret = strings.TrimSpace(parts[1])
}
}
}
if sourceClientID != "" && sourceClientSecret != "" && destClientID != "" && destClientSecret != "" {
break
}
}
if destClientID != "" && destClientSecret != "" {
return destClientID, destClientSecret
}
if sourceClientID != "" && sourceClientSecret != "" {
return sourceClientID, sourceClientSecret
}
return "", ""
}