From 31871bd16e349dee7f98d72eb6cd39ed2b58308a Mon Sep 17 00:00:00 2001 From: StarFleetCPTN Date: Wed, 16 Apr 2025 17:18:53 -0700 Subject: [PATCH] feat: Implement storage provider management functionality - Added new routes and handlers for managing storage providers, including creation, editing, and deletion. - Introduced a new StorageProvider form component for user input. - Enhanced the database schema to support storage provider references in transfer configurations. - Implemented encryption for sensitive fields in storage provider data. - Added tests for storage provider API endpoints and integration with the database. - Updated frontend components to support storage provider selection and testing. --- .air.toml | 2 +- components/config_form.templ | 502 +++++----- components/layout.templ | 8 + components/providers/common/common.templ | 190 +++- components/storage_provider_form.templ | 404 ++++++++ components/storage_providers.templ | 568 +++++++++++ go.mod | 3 +- go.sum | 29 +- internal/api/api.go | 265 +++++ internal/api/storage_provider_api_test.go | 612 ++++++++++++ .../api/tests/storage_provider_api_test.go | 393 ++++++++ internal/db/db.go | 64 +- .../db/middleware/encryption_middleware.go | 331 +++++++ .../middleware/encryption_middleware_test.go | 253 +++++ internal/db/migrate_provider_data.go | 922 ++++++++++++++++++ .../migrations/014_add_storage_providers.go | 66 ++ ...15_add_provider_refs_to_transfer_config.go | 55 ++ internal/db/migrations/migrations.go | 2 + internal/db/storage_provider.go | 205 ++++ internal/db/storage_provider_connector.go | 42 + .../db/storage_provider_integration_test.go | 416 ++++++++ internal/db/storage_provider_store.go | 83 ++ internal/db/storage_provider_validator.go | 241 +++++ .../db/storage_provider_validator_test.go | 141 +++ internal/db/transfer_config.go | 544 +++++++++++ .../transfer_config_storage_provider_test.go | 367 +++++++ internal/db/transfer_config_store.go | 571 ++++++++++- internal/encryption/README.md | 67 ++ internal/encryption/audit/audit.go | 400 ++++++++ .../encryption/audit/key_rotation_utility.go | 512 ++++++++++ internal/encryption/audit/monitor.go | 271 +++++ internal/encryption/audit/monitor_test.go | 99 ++ .../audit/security_test_framework.go | 563 +++++++++++ .../audit/security_test_framework_test.go | 196 ++++ internal/encryption/constants.go | 23 + internal/encryption/credential_encryptor.go | 274 ++++++ .../encryption/credential_encryptor_test.go | 329 +++++++ internal/encryption/crypto.go | 194 ++++ internal/encryption/crypto_test.go | 274 ++++++ internal/encryption/encryption.go | 200 ++++ internal/encryption/encryption_test.go | 177 ++++ internal/encryption/keymanager/keymanager.go | 147 +++ .../encryption/keymanager/keymanager_test.go | 148 +++ .../encryption/keyrotation/key_rotator.go | 286 ++++++ .../keyrotation/key_rotator_test.go | 512 ++++++++++ .../keyrotation/rotation_utility.go | 505 ++++++++++ internal/encryption/sanitize.go | 103 ++ internal/encryptionsecurity/README.md | 189 ++++ .../encryptionsecurity/security_framework.go | 426 ++++++++ internal/scheduler/transfer_executor.go | 230 ++++- internal/storage/connector_service.go | 207 ++++ .../web/e2e/storage_provider_flow_test.go | 335 +++++++ .../web/e2e/storage_provider_security_test.go | 377 +++++++ internal/web/handlers/config_handlers.go | 566 +++++++---- .../handlers/provider_type_retrieval_test.go | 153 +++ internal/web/handlers/routes.go | 14 + .../web/handlers/storage_provider_handlers.go | 498 ++++++++++ .../storage_provider_handlers_test.go | 714 ++++++++++++++ static/js/storage-providers.js | 168 ++++ 59 files changed, 15842 insertions(+), 594 deletions(-) create mode 100644 components/storage_provider_form.templ create mode 100644 components/storage_providers.templ create mode 100644 internal/api/storage_provider_api_test.go create mode 100644 internal/api/tests/storage_provider_api_test.go create mode 100644 internal/db/middleware/encryption_middleware.go create mode 100644 internal/db/middleware/encryption_middleware_test.go create mode 100644 internal/db/migrate_provider_data.go create mode 100644 internal/db/migrations/014_add_storage_providers.go create mode 100644 internal/db/migrations/015_add_provider_refs_to_transfer_config.go create mode 100644 internal/db/storage_provider.go create mode 100644 internal/db/storage_provider_connector.go create mode 100644 internal/db/storage_provider_integration_test.go create mode 100644 internal/db/storage_provider_store.go create mode 100644 internal/db/storage_provider_validator.go create mode 100644 internal/db/storage_provider_validator_test.go create mode 100644 internal/db/transfer_config_storage_provider_test.go create mode 100644 internal/encryption/README.md create mode 100644 internal/encryption/audit/audit.go create mode 100644 internal/encryption/audit/key_rotation_utility.go create mode 100644 internal/encryption/audit/monitor.go create mode 100644 internal/encryption/audit/monitor_test.go create mode 100644 internal/encryption/audit/security_test_framework.go create mode 100644 internal/encryption/audit/security_test_framework_test.go create mode 100644 internal/encryption/constants.go create mode 100644 internal/encryption/credential_encryptor.go create mode 100644 internal/encryption/credential_encryptor_test.go create mode 100644 internal/encryption/crypto.go create mode 100644 internal/encryption/crypto_test.go create mode 100644 internal/encryption/encryption.go create mode 100644 internal/encryption/encryption_test.go create mode 100644 internal/encryption/keymanager/keymanager.go create mode 100644 internal/encryption/keymanager/keymanager_test.go create mode 100644 internal/encryption/keyrotation/key_rotator.go create mode 100644 internal/encryption/keyrotation/key_rotator_test.go create mode 100644 internal/encryption/keyrotation/rotation_utility.go create mode 100644 internal/encryption/sanitize.go create mode 100644 internal/encryptionsecurity/README.md create mode 100644 internal/encryptionsecurity/security_framework.go create mode 100644 internal/storage/connector_service.go create mode 100644 internal/web/e2e/storage_provider_flow_test.go create mode 100644 internal/web/e2e/storage_provider_security_test.go create mode 100644 internal/web/handlers/provider_type_retrieval_test.go create mode 100644 internal/web/handlers/storage_provider_handlers.go create mode 100644 internal/web/handlers/storage_provider_handlers_test.go create mode 100644 static/js/storage-providers.js diff --git a/.air.toml b/.air.toml index 77b3a6b..6ee6a0b 100644 --- a/.air.toml +++ b/.air.toml @@ -7,7 +7,7 @@ args_bin = [] bin = "./tmp/main" cmd = "templ generate && go build -o ./tmp/main ." delay = 1000 -exclude_dir = ["assets", "tmp", "vendor", "testdata", "node_modules"] +exclude_dir = ["assets", "tmp", "vendor", "testdata", "node_modules", "docs"] exclude_file = [] exclude_regex = ["_test.go", "_templ.go"] exclude_unchanged = false diff --git a/components/config_form.templ b/components/config_form.templ index 547a2c1..ff377fe 100644 --- a/components/config_form.templ +++ b/components/config_form.templ @@ -16,6 +16,9 @@ type ConfigFormData struct { InitialCommand *db.RcloneCommand SelectedFlagsMap map[uint]bool SelectedFlagValues map[uint]string + // Add source and destination providers + SourceProviders []db.StorageProvider + DestinationProviders []db.StorageProvider } func getConfigFormTitle(isNew bool) string { @@ -92,6 +95,13 @@ func getInitialData(config *db.TransferConfig) string { useBuiltinAuthSource := true useBuiltinAuthDest := true + // Provider configuration + useSourceProvider := false + sourceProviderId := uint(0) + + useDestinationProvider := false + destinationProviderId := uint(0) + // If editing an existing config, populate with those values if config != nil { name = config.Name @@ -183,6 +193,17 @@ func getInitialData(config *db.TransferConfig) string { } else if destClientId != "" || destClientSecret != "" { useBuiltinAuthDest = false } + + // Provider reference fields + if config.IsUsingSourceProviderReference() { + useSourceProvider = true + sourceProviderId = *config.SourceProviderID + } + + if config.IsUsingDestinationProviderReference() { + useDestinationProvider = true + destinationProviderId = *config.DestinationProviderID + } } // Return the JSON-formatted string with all the data, add new path validation states @@ -212,6 +233,9 @@ func getInitialData(config *db.TransferConfig) string { sourceStartYear: %d, sourceIncludeArchived: %v, + useSourceProvider: %v, + sourceProviderId: %d, + filePattern: '%s', outputPattern: '%s', @@ -239,6 +263,9 @@ func getInitialData(config *db.TransferConfig) string { destStartYear: %d, destIncludeArchived: %v, + useDestinationProvider: %v, + destinationProviderId: %d, + useBuiltinAuthSource: %v, useBuiltinAuthDest: %v, @@ -347,11 +374,13 @@ func getInitialData(config *db.TransferConfig) string { sourceBucket, sourceRegion, sourceAccessKey, sourceSecretKey, sourceEndpoint, sourceShare, sourceDomain, sourcePassiveMode, sourceClientId, sourceClientSecret, sourceDriveId, sourceTeamDrive, sourceReadOnly, sourceStartYear, sourceIncludeArchived, + useSourceProvider, sourceProviderId, filePattern, outputPattern, destinationType, destinationPath, destHost, destPort, destUser, destPassword, destKeyFile, destAuthType, destBucket, destRegion, destAccessKey, destSecretKey, destEndpoint, destShare, destDomain, destPassiveMode, destClientId, destClientSecret, destDriveId, destTeamDrive, destReadOnly, destStartYear, destIncludeArchived, + useDestinationProvider, destinationProviderId, useBuiltinAuthSource, useBuiltinAuthDest, archivePath, archiveEnabled, deleteAfterTransfer, skipProcessedFiles, maxConcurrentTransfers, rcloneFlags, commandId, commandFlags) @@ -422,6 +451,10 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { maxConcurrentTransfers = 1; } + // Initialize provider selection states + useSourceProvider = useSourceProvider || false; + useDestinationProvider = useDestinationProvider || false; + // Initialize command requirements updateCommandRequirements(); })" @@ -512,8 +545,9 @@ templ ConfigForm(ctx context.Context, data ConfigFormData) { - @common.SourceSelection() + @common.SourceSelection(data.SourceProviders) +
-
- -