mirror of
https://github.com/StarFleetCPTN/GoMFT.git
synced 2026-11-19 19:03:29 +01:00
feat: Add multi-threaded file transfers and enhanced logging support
- Implement concurrent file transfer processing with configurable concurrency - Add new `max_concurrent_transfers` column to transfer configurations - Enhance scheduler to support multi-threaded file transfers - Introduce advanced logging system with rotation and configurable log levels - Update Docker Compose and documentation with new logging configuration options - Modify directory structure to separate data, backups, and logs - Add environment variables for comprehensive logging control - Improve error handling and logging in file transfer processes
This commit is contained in:
1 parent
16b234b2d3
commit
ed81b2c9be
13 files changed
+693
-329
No files matched your search
+4
-1
@@ -68,11 +68,14 @@ tmp/
|
|||||||
# Ignore the configs directory
|
# Ignore the configs directory
|
||||||
configs/
|
configs/
|
||||||
|
|
||||||
|
# Ignore the backups directory
|
||||||
|
backups/
|
||||||
|
|
||||||
# Ignore Dirs
|
# Ignore Dirs
|
||||||
/source/
|
/source/
|
||||||
/destination/
|
/destination/
|
||||||
|
|
||||||
/archive/
|
/archive/
|
||||||
|
|
||||||
|
|
||||||
# Ignore binaries
|
# Ignore binaries
|
||||||
gomft
|
gomft
|
||||||
+2
-2
@@ -47,8 +47,8 @@ COPY --from=builder /usr/local/bin/rclone /usr/local/bin/rclone
|
|||||||
COPY static/ /app/static/
|
COPY static/ /app/static/
|
||||||
COPY components/ /app/components/
|
COPY components/ /app/components/
|
||||||
|
|
||||||
# Create data directory
|
# Create data and backup directories
|
||||||
RUN mkdir -p /app/data/gomft
|
RUN mkdir -p /app/data /app/backups
|
||||||
|
|
||||||
# Set executable permissions
|
# Set executable permissions
|
||||||
RUN chmod +x /app/gomft
|
RUN chmod +x /app/gomft
|
||||||
|
|||||||
@@ -52,6 +52,12 @@ GoMFT is a web-based managed file transfer application built with Go, leveraging
|
|||||||
- Advanced search interface with multiple criteria
|
- Advanced search interface with multiple criteria
|
||||||
- Bulk management and record deletion capabilities
|
- Bulk management and record deletion capabilities
|
||||||
- Responsive design with mobile-friendly interface
|
- Responsive design with mobile-friendly interface
|
||||||
|
- **Multi-threaded File Transfers**: Significantly improve performance with concurrent file processing:
|
||||||
|
- Configurable number of concurrent transfers (1-32) per job
|
||||||
|
- Automatic queue management to prevent system overload
|
||||||
|
- Independent configuration for each transfer job
|
||||||
|
- Optimized for both high-volume small files and large file transfers
|
||||||
|
- Maximizes bandwidth utilization for cloud storage providers
|
||||||
- **Web Interface**: User-friendly interface for managing transfers, built with Templ components
|
- **Web Interface**: User-friendly interface for managing transfers, built with Templ components
|
||||||
- **File Pattern Matching**: Support for file patterns to filter files during transfers
|
- **File Pattern Matching**: Support for file patterns to filter files during transfers
|
||||||
- **File Output Patterns**: Dynamic naming of destination files using patterns with date variables
|
- **File Output Patterns**: Dynamic naming of destination files using patterns with date variables
|
||||||
@@ -106,6 +112,7 @@ docker run -d \
|
|||||||
--name gomft \
|
--name gomft \
|
||||||
-p 8080:8080 \
|
-p 8080:8080 \
|
||||||
-v /path/to/data:/app/data \
|
-v /path/to/data:/app/data \
|
||||||
|
-v /path/to/backups:/app/backups \
|
||||||
starfleetcptn/gomft:latest
|
starfleetcptn/gomft:latest
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -126,12 +133,12 @@ services:
|
|||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./backups:/app/data/gomft/backups
|
- ./backups:/app/backups
|
||||||
environment:
|
environment:
|
||||||
- TZ=UTC
|
- TZ=UTC
|
||||||
- SERVER_ADDRESS=:8080
|
- SERVER_ADDRESS=:8080
|
||||||
- DATA_DIR=/app/data/gomft
|
- DATA_DIR=/app/data
|
||||||
- BACKUP_DIR=/app/data/gomft/backups
|
- BACKUP_DIR=/app/backups
|
||||||
- JWT_SECRET=change_this_to_a_secure_random_string
|
- JWT_SECRET=change_this_to_a_secure_random_string
|
||||||
- BASE_URL=http://localhost:8080
|
- BASE_URL=http://localhost:8080
|
||||||
- EMAIL_ENABLED=true
|
- EMAIL_ENABLED=true
|
||||||
@@ -143,6 +150,13 @@ services:
|
|||||||
- EMAIL_REQUIRE_AUTH=true
|
- EMAIL_REQUIRE_AUTH=true
|
||||||
- EMAIL_USERNAME=smtp_username
|
- EMAIL_USERNAME=smtp_username
|
||||||
- EMAIL_PASSWORD=smtp_password
|
- EMAIL_PASSWORD=smtp_password
|
||||||
|
# Logging configuration
|
||||||
|
- GOMFT_LOGS_DIR=/app/data/logs
|
||||||
|
- GOMFT_LOG_MAX_SIZE=10
|
||||||
|
- GOMFT_LOG_MAX_BACKUPS=5
|
||||||
|
- GOMFT_LOG_MAX_AGE=30
|
||||||
|
- GOMFT_LOG_COMPRESS=true
|
||||||
|
- GOMFT_LOG_LEVEL=info
|
||||||
```
|
```
|
||||||
|
|
||||||
Alternatively, you can mount your own .env file to the container:
|
Alternatively, you can mount your own .env file to the container:
|
||||||
@@ -158,7 +172,7 @@ services:
|
|||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
volumes:
|
volumes:
|
||||||
- ./data:/app/data
|
- ./data:/app/data
|
||||||
- ./backups:/app/data/gomft/backups
|
- ./backups:/app/backups
|
||||||
- ./.env:/app/.env
|
- ./.env:/app/.env
|
||||||
environment:
|
environment:
|
||||||
- TZ=UTC
|
- TZ=UTC
|
||||||
@@ -178,8 +192,8 @@ GoMFT uses an environment file located at `.env` in the root directory of the ap
|
|||||||
|
|
||||||
```
|
```
|
||||||
SERVER_ADDRESS=:8080
|
SERVER_ADDRESS=:8080
|
||||||
DATA_DIR=./data/gomft
|
DATA_DIR=/app/data
|
||||||
BACKUP_DIR=./data/gomft/backups
|
BACKUP_DIR=/app/backups
|
||||||
JWT_SECRET=change_this_to_a_secure_random_string
|
JWT_SECRET=change_this_to_a_secure_random_string
|
||||||
BASE_URL=http://localhost:8080
|
BASE_URL=http://localhost:8080
|
||||||
|
|
||||||
@@ -199,7 +213,7 @@ EMAIL_PASSWORD=smtp_password
|
|||||||
### Configuration Options
|
### Configuration Options
|
||||||
|
|
||||||
- `SERVER_ADDRESS`: The address and port to run the server on
|
- `SERVER_ADDRESS`: The address and port to run the server on
|
||||||
- `DATA_DIR`: Directory for storing application data
|
- `DATA_DIR`: Directory for storing application data (database and configs)
|
||||||
- `BACKUP_DIR`: Directory for storing database backups
|
- `BACKUP_DIR`: Directory for storing database backups
|
||||||
- `JWT_SECRET`: Secret key for JWT token generation
|
- `JWT_SECRET`: Secret key for JWT token generation
|
||||||
- `BASE_URL`: Base URL for generating links in emails (e.g., password reset links)
|
- `BASE_URL`: Base URL for generating links in emails (e.g., password reset links)
|
||||||
@@ -215,6 +229,22 @@ EMAIL_PASSWORD=smtp_password
|
|||||||
- `EMAIL_ENABLE_TLS`: Set to `true` to use TLS for secure email transmission
|
- `EMAIL_ENABLE_TLS`: Set to `true` to use TLS for secure email transmission
|
||||||
- `EMAIL_REQUIRE_AUTH`: Set to `true` to require authentication for SMTP connections, or `false` for servers that don't need authentication
|
- `EMAIL_REQUIRE_AUTH`: Set to `true` to require authentication for SMTP connections, or `false` for servers that don't need authentication
|
||||||
|
|
||||||
|
### Logging Configuration
|
||||||
|
|
||||||
|
GoMFT provides configurable logging with rotation support through the following environment variables:
|
||||||
|
|
||||||
|
- `GOMFT_LOGS_DIR`: Directory where log files are stored (default: `./data/logs`)
|
||||||
|
- `GOMFT_LOG_MAX_SIZE`: Maximum size in megabytes for each log file before rotation (default: `10`)
|
||||||
|
- `GOMFT_LOG_MAX_BACKUPS`: Number of old log files to retain (default: `5`)
|
||||||
|
- `GOMFT_LOG_MAX_AGE`: Maximum number of days to retain old log files (default: `30`)
|
||||||
|
- `GOMFT_LOG_COMPRESS`: Whether to compress rotated log files (default: `true`)
|
||||||
|
- `GOMFT_LOG_LEVEL`: Controls verbosity level of logging (values: `error`, `info`, `debug`, default: `info`)
|
||||||
|
- `error`: Only show errors and critical issues
|
||||||
|
- `info`: Show errors and general operational information (default)
|
||||||
|
- `debug`: Show all messages including detailed debugging information
|
||||||
|
|
||||||
|
Log files contain detailed information about file transfers, job execution, and system operations, which can be useful for troubleshooting and auditing.
|
||||||
|
|
||||||
## Usage
|
## Usage
|
||||||
|
|
||||||
1. Start the server:
|
1. Start the server:
|
||||||
@@ -233,6 +263,11 @@ EMAIL_PASSWORD=smtp_password
|
|||||||
- Navigate to "Transfer Configs" section
|
- Navigate to "Transfer Configs" section
|
||||||
- Configure source and destination locations with connection details
|
- Configure source and destination locations with connection details
|
||||||
- Set file patterns and archive options as needed
|
- Set file patterns and archive options as needed
|
||||||
|
- Configure performance settings:
|
||||||
|
- Set "Concurrent Transfers" slider to optimize throughput
|
||||||
|
- Use higher values (8-16) for many small files or fast networks
|
||||||
|
- Use lower values (1-4) for large files or limited bandwidth
|
||||||
|
- Consider source/destination system capabilities when setting
|
||||||
|
|
||||||
5. Create jobs using your configurations:
|
5. Create jobs using your configurations:
|
||||||
- Navigate to "Jobs" section
|
- Navigate to "Jobs" section
|
||||||
@@ -294,8 +329,17 @@ User management features:
|
|||||||
- File patterns for filtering (e.g., `*.txt`, `data_*.csv`)
|
- File patterns for filtering (e.g., `*.txt`, `data_*.csv`)
|
||||||
- Output patterns for dynamic naming
|
- Output patterns for dynamic naming
|
||||||
- Archive options for transferred files
|
- Archive options for transferred files
|
||||||
|
- Skip already processed files to avoid duplicates
|
||||||
|
- Concurrent file transfers (configurable per job)
|
||||||
|
|
||||||
4. **Schedule Options**:
|
4. **Performance Options**:
|
||||||
|
- **Multi-threaded File Transfers**: Process multiple files simultaneously for higher throughput
|
||||||
|
- Configurable concurrency level (1-32 concurrent transfers)
|
||||||
|
- Per-job concurrency settings to optimize for different storage types
|
||||||
|
- Automatic transfer queue management to prevent overloading systems
|
||||||
|
- Adaptive processing based on source/destination capabilities
|
||||||
|
|
||||||
|
5. **Schedule Options**:
|
||||||
- Cron expressions for flexible scheduling
|
- Cron expressions for flexible scheduling
|
||||||
- Manual execution
|
- Manual execution
|
||||||
- Enable/disable schedules
|
- Enable/disable schedules
|
||||||
@@ -379,3 +423,23 @@ air
|
|||||||
## License
|
## License
|
||||||
|
|
||||||
MIT License - see LICENSE file for details
|
MIT License - see LICENSE file for details
|
||||||
|
|
||||||
|
## Directory Structure
|
||||||
|
|
||||||
|
GoMFT uses the following directory structure:
|
||||||
|
|
||||||
|
- `/app/data`: Main application data directory
|
||||||
|
- Contains the SQLite database (`gomft.db`)
|
||||||
|
- Contains rclone configurations in `/app/data/configs`
|
||||||
|
- Contains log files in `/app/data/logs`
|
||||||
|
- `/app/backups`: Database backup directory
|
||||||
|
|
||||||
|
When using Docker, you should mount volumes to these locations:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
volumes:
|
||||||
|
- /host/path/data:/app/data # For all application data
|
||||||
|
- /host/path/backups:/app/backups # For database backups
|
||||||
|
```
|
||||||
|
|
||||||
|
These paths can be customized using the environment variables `DATA_DIR`, `BACKUP_DIR`, and `GOMFT_LOGS_DIR`.
|
||||||
@@ -73,6 +73,7 @@ func getInitialData(config *db.TransferConfig) string {
|
|||||||
archiveEnabled := false
|
archiveEnabled := false
|
||||||
deleteAfterTransfer := false
|
deleteAfterTransfer := false
|
||||||
skipProcessedFiles := true
|
skipProcessedFiles := true
|
||||||
|
maxConcurrentTransfers := 4
|
||||||
rcloneFlags := ""
|
rcloneFlags := ""
|
||||||
|
|
||||||
// If editing an existing config, populate with those values
|
// If editing an existing config, populate with those values
|
||||||
@@ -131,6 +132,7 @@ func getInitialData(config *db.TransferConfig) string {
|
|||||||
archiveEnabled = config.ArchiveEnabled
|
archiveEnabled = config.ArchiveEnabled
|
||||||
deleteAfterTransfer = config.DeleteAfterTransfer
|
deleteAfterTransfer = config.DeleteAfterTransfer
|
||||||
skipProcessedFiles = config.SkipProcessedFiles
|
skipProcessedFiles = config.SkipProcessedFiles
|
||||||
|
maxConcurrentTransfers = config.MaxConcurrentTransfers
|
||||||
rcloneFlags = config.RcloneFlags
|
rcloneFlags = config.RcloneFlags
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -186,6 +188,7 @@ func getInitialData(config *db.TransferConfig) string {
|
|||||||
archiveEnabled: %v,
|
archiveEnabled: %v,
|
||||||
deleteAfterTransfer: %v,
|
deleteAfterTransfer: %v,
|
||||||
skipProcessedFiles: %v,
|
skipProcessedFiles: %v,
|
||||||
|
maxConcurrentTransfers: %d,
|
||||||
rcloneFlags: '%s',
|
rcloneFlags: '%s',
|
||||||
}`,
|
}`,
|
||||||
name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
|
name, sourceType, sourcePath, sourceHost, sourcePort, sourceUser, sourcePassword, sourceKeyFile, sourceAuthType,
|
||||||
@@ -195,7 +198,7 @@ func getInitialData(config *db.TransferConfig) string {
|
|||||||
destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
|
destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType,
|
||||||
destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
|
destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode,
|
||||||
destClientId, destClientSecret, destDriveId, destTeamDrive,
|
destClientId, destClientSecret, destDriveId, destTeamDrive,
|
||||||
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, rcloneFlags)
|
archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags)
|
||||||
}
|
}
|
||||||
|
|
||||||
templ ConfigForm(ctx context.Context, data ConfigFormData) {
|
templ ConfigForm(ctx context.Context, data ConfigFormData) {
|
||||||
|
|||||||
@@ -141,6 +141,21 @@ templ ArchiveOptions() {
|
|||||||
Files with the same hash that have been successfully processed before will be skipped
|
Files with the same hash that have been successfully processed before will be skipped
|
||||||
</p>
|
</p>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<div class="mb-4">
|
||||||
|
<label for="max_concurrent_transfers" class="block text-sm font-medium text-gray-700 mb-1">Concurrent Transfers</label>
|
||||||
|
<div class="flex items-center">
|
||||||
|
<input type="range" id="max_concurrent_transfers" name="max_concurrent_transfers"
|
||||||
|
class="form-range w-2/3 h-2 bg-gray-200 rounded-lg appearance-none cursor-pointer"
|
||||||
|
min="1" max="20" step="1"
|
||||||
|
x-model="maxConcurrentTransfers"
|
||||||
|
/>
|
||||||
|
<span class="ml-4 text-gray-700" x-text="maxConcurrentTransfers"></span>
|
||||||
|
</div>
|
||||||
|
<p class="mt-1 text-xs text-gray-500">
|
||||||
|
Number of files to transfer simultaneously (higher values may improve performance but increase resource usage)
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+8
-3
@@ -8,14 +8,17 @@ services:
|
|||||||
ports:
|
ports:
|
||||||
- "8080:8080"
|
- "8080:8080"
|
||||||
volumes:
|
volumes:
|
||||||
# Persist data directory for SQLite database and configurations
|
# Main data directory - contains DB and configs
|
||||||
- gomft-data:/app/data
|
- gomft-data:/app/data
|
||||||
|
# Separate backups directory
|
||||||
|
- gomft-backups:/app/backups
|
||||||
# For development, you can mount the source code
|
# For development, you can mount the source code
|
||||||
# - .:/app
|
# - .:/app
|
||||||
environment:
|
environment:
|
||||||
- TZ=UTC
|
- TZ=UTC
|
||||||
# Add any environment variables needed for configuration
|
- DATA_DIR=/app/data
|
||||||
# - GOMFT_DB_PATH=/app/data/gomft.db
|
- BACKUP_DIR=/app/backups
|
||||||
|
- GOMFT_LOGS_DIR=/app/data/logs
|
||||||
# - GOMFT_LOG_LEVEL=info
|
# - GOMFT_LOG_LEVEL=info
|
||||||
networks:
|
networks:
|
||||||
- gomft-network
|
- gomft-network
|
||||||
@@ -27,3 +30,5 @@ networks:
|
|||||||
volumes:
|
volumes:
|
||||||
gomft-data:
|
gomft-data:
|
||||||
driver: local
|
driver: local
|
||||||
|
gomft-backups:
|
||||||
|
driver: local
|
||||||
@@ -44,6 +44,7 @@ require (
|
|||||||
golang.org/x/sys v0.30.0 // indirect
|
golang.org/x/sys v0.30.0 // indirect
|
||||||
golang.org/x/text v0.22.0 // indirect
|
golang.org/x/text v0.22.0 // indirect
|
||||||
google.golang.org/protobuf v1.36.5 // indirect
|
google.golang.org/protobuf v1.36.5 // indirect
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||||
modernc.org/libc v1.22.5 // indirect
|
modernc.org/libc v1.22.5 // indirect
|
||||||
modernc.org/mathutil v1.5.0 // indirect
|
modernc.org/mathutil v1.5.0 // indirect
|
||||||
|
|||||||
@@ -105,6 +105,8 @@ google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwl
|
|||||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
|
||||||
|
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -35,8 +34,8 @@ func Load() (*Config, error) {
|
|||||||
// Default configuration
|
// Default configuration
|
||||||
cfg := &Config{
|
cfg := &Config{
|
||||||
ServerAddress: ":8080",
|
ServerAddress: ":8080",
|
||||||
DataDir: filepath.Join("./data", "gomft"),
|
DataDir: "./data",
|
||||||
BackupDir: filepath.Join("./data", "gomft", "backups"),
|
BackupDir: "./backups",
|
||||||
JWTSecret: "change_this_to_a_secure_random_string",
|
JWTSecret: "change_this_to_a_secure_random_string",
|
||||||
BaseURL: "http://localhost:8080",
|
BaseURL: "http://localhost:8080",
|
||||||
Email: EmailConfig{
|
Email: EmailConfig{
|
||||||
|
|||||||
+28
-11
@@ -98,15 +98,16 @@ type TransferConfig struct {
|
|||||||
DestDriveID string `form:"dest_drive_id"` // For OneDrive
|
DestDriveID string `form:"dest_drive_id"` // For OneDrive
|
||||||
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
|
DestTeamDrive string `form:"dest_team_drive"` // For Google Drive
|
||||||
// General fields
|
// General fields
|
||||||
ArchivePath string `form:"archive_path"`
|
ArchivePath string `form:"archive_path"`
|
||||||
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
|
ArchiveEnabled bool `gorm:"default:false" form:"archive_enabled"`
|
||||||
RcloneFlags string `form:"rclone_flags"`
|
RcloneFlags string `form:"rclone_flags"`
|
||||||
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
|
DeleteAfterTransfer bool `gorm:"default:false" form:"delete_after_transfer"`
|
||||||
SkipProcessedFiles bool `gorm:"default:true" form:"skip_processed_files"`
|
SkipProcessedFiles bool `gorm:"default:true" form:"skip_processed_files"`
|
||||||
CreatedBy uint
|
MaxConcurrentTransfers int `gorm:"default:4" form:"max_concurrent_transfers"` // Number of concurrent file transfers
|
||||||
User User `gorm:"foreignkey:CreatedBy"`
|
CreatedBy uint
|
||||||
CreatedAt time.Time
|
User User `gorm:"foreignkey:CreatedBy"`
|
||||||
UpdatedAt time.Time
|
CreatedAt time.Time
|
||||||
|
UpdatedAt time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
type Job struct {
|
type Job struct {
|
||||||
@@ -381,7 +382,14 @@ func (db *DB) DeleteFileMetadata(id uint) error {
|
|||||||
|
|
||||||
// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
|
// GetConfigRclonePath returns the path to the rclone config file for a given transfer config
|
||||||
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
|
func (db *DB) GetConfigRclonePath(config *TransferConfig) string {
|
||||||
return filepath.Join("configs", fmt.Sprintf("config_%d.conf", config.ID))
|
// Get data directory from environment or use default
|
||||||
|
dataDir := os.Getenv("DATA_DIR")
|
||||||
|
if dataDir == "" {
|
||||||
|
dataDir = "./data"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Store configs in the data directory
|
||||||
|
return filepath.Join(dataDir, "configs", fmt.Sprintf("config_%d.conf", config.ID))
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
|
// GetSkipProcessedFilesValue gets the current value of SkipProcessedFiles for a config
|
||||||
@@ -397,8 +405,11 @@ func (db *DB) GetSkipProcessedFilesValue(configID uint) (bool, error) {
|
|||||||
func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
|
func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
|
||||||
configPath := db.GetConfigRclonePath(config)
|
configPath := db.GetConfigRclonePath(config)
|
||||||
|
|
||||||
|
// Get the directory part of the path
|
||||||
|
configDir := filepath.Dir(configPath)
|
||||||
|
|
||||||
// Ensure configs directory exists
|
// Ensure configs directory exists
|
||||||
if err := os.MkdirAll("configs", 0755); err != nil {
|
if err := os.MkdirAll(configDir, 0755); err != nil {
|
||||||
return fmt.Errorf("failed to create configs directory: %v", err)
|
return fmt.Errorf("failed to create configs directory: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -792,3 +803,9 @@ func (db *DB) GenerateRcloneConfig(config *TransferConfig) error {
|
|||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (db *DB) GetActiveJobs() ([]Job, error) {
|
||||||
|
var jobs []Job
|
||||||
|
err := db.Preload("Config").Where("enabled = ?", true).Find(&jobs).Error
|
||||||
|
return jobs, err
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package migrations
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/go-gormigrate/gormigrate/v2"
|
||||||
|
"gorm.io/gorm"
|
||||||
|
)
|
||||||
|
|
||||||
|
// AddMaxConcurrentTransfersColumn adds the max_concurrent_transfers column to transfer_configs table
|
||||||
|
func AddMaxConcurrentTransfersColumn() *gormigrate.Migration {
|
||||||
|
return &gormigrate.Migration{
|
||||||
|
ID: "20250311_add_max_concurrent_transfers",
|
||||||
|
Migrate: func(tx *gorm.DB) error {
|
||||||
|
// Add max_concurrent_transfers column with default value of 4
|
||||||
|
return tx.Exec("ALTER TABLE transfer_configs ADD COLUMN max_concurrent_transfers INTEGER DEFAULT 4").Error
|
||||||
|
},
|
||||||
|
Rollback: func(tx *gorm.DB) error {
|
||||||
|
// Drop the column if needed
|
||||||
|
return tx.Exec("ALTER TABLE transfer_configs DROP COLUMN max_concurrent_transfers").Error
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,6 +12,7 @@ func InitMigrations(db *gorm.DB) *gormigrate.Gormigrate {
|
|||||||
AddDeleteAfterTransferColumn(),
|
AddDeleteAfterTransferColumn(),
|
||||||
AddCloudStorageFields(),
|
AddCloudStorageFields(),
|
||||||
AddSkipProcessedFilesColumn(),
|
AddSkipProcessedFilesColumn(),
|
||||||
|
AddMaxConcurrentTransfersColumn(),
|
||||||
}
|
}
|
||||||
|
|
||||||
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
|
return gormigrate.New(db, gormigrate.DefaultOptions, migrations)
|
||||||
|
|||||||
+533
-300
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user