From f9907f43261bba5588bb67cdda4d4b2ba6299201 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 29 Jan 2026 22:49:54 -0800 Subject: [PATCH] adjust endpoint in aws CLI --- AWS-CLI-with-SeaweedFS.md | 26 ++++++++++-------- Amazon-S3-API.md | 27 ++++++++++--------- Quick-Start-with-weed-mini.md | 14 ++++++---- S3-API-FAQ.md | 10 +++---- S3-CORS.md | 9 ++++--- S3-Credentials.md | 5 ++-- S3-Object-Lock-and-Retention.md | 45 +++++++++++++++++-------------- S3-Object-Versioning.md | 19 +++++++------ S3-Policy-Variables.md | 8 +++--- Server-Side-Encryption-SSE-C.md | 21 ++++++++------- Server-Side-Encryption-SSE-KMS.md | 9 ++++--- Server-Side-Encryption.md | 9 ++++--- 12 files changed, 116 insertions(+), 86 deletions(-) diff --git a/AWS-CLI-with-SeaweedFS.md b/AWS-CLI-with-SeaweedFS.md index c5ce118..b2bd0a2 100644 --- a/AWS-CLI-with-SeaweedFS.md +++ b/AWS-CLI-with-SeaweedFS.md @@ -16,7 +16,11 @@ Default output format [None]: Also, make sure using AWS signature version 4: ``` $ aws configure set default.s3.signature_version s3v4 +``` +And configure the S3 endpoint: +```bash +export S3_ENDPOINT=http://localhost:8333 ``` ### Reverse proxy sub-path configuration @@ -39,31 +43,31 @@ handle_path /s3/* { ### Execute commands ``` # list buckets -$ aws --endpoint-url http://localhost:8333 s3 ls +$ aws --endpoint-url $S3_ENDPOINT s3 ls 2019-01-02 01:59:25 newbucket # list files inside the bucket -$ aws --endpoint-url http://localhost:8333 s3 ls s3://newbucket +$ aws --endpoint-url $S3_ENDPOINT s3 ls s3://newbucket 2019-01-02 12:52:44 6804 password # make a bucket -$ aws --endpoint-url http://localhost:8333 s3 mb s3://newbucket3 +$ aws --endpoint-url $S3_ENDPOINT s3 mb s3://newbucket3 make_bucket: newbucket3 # add an object -$ aws --endpoint-url http://localhost:8333 s3 cp /etc/passwd s3://newbucket3 +$ aws --endpoint-url $S3_ENDPOINT s3 cp /etc/passwd s3://newbucket3 upload: ../../../../../etc/passwd to s3://newbucket3/passwd # copy an object -$ aws --endpoint-url http://localhost:8333 s3 cp s3://newbucket3/passwd s3://newbucket3/passwd.txt +$ aws --endpoint-url $S3_ENDPOINT s3 cp s3://newbucket3/passwd s3://newbucket3/passwd.txt copy: s3://newbucket3/passwd to s3://newbucket3/passwd.txt # remove an object -$ aws --endpoint-url http://localhost:8333 s3 rm s3://newbucket3/passwd +$ aws --endpoint-url $S3_ENDPOINT s3 rm s3://newbucket3/passwd delete: s3://newbucket3/passwd # remove a bucket -$ aws --endpoint-url http://localhost:8333 s3 rb s3://newbucket3 +$ aws --endpoint-url $S3_ENDPOINT s3 rb s3://newbucket3 remove_bucket: newbucket3 ``` @@ -76,7 +80,7 @@ If [authentication](https://github.com/seaweedfs/seaweedfs/wiki/Amazon-S3-API#au ``` # presign url, default to 1 hour -$ aws --endpoint-url http://localhost:8333 s3 presign s3://newbucket/t.txt +$ aws --endpoint-url $S3_ENDPOINT s3 presign s3://newbucket/t.txt http://localhost:8333/newbucket/t.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=some_access_key1%2F20200726%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20200726T161749Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=e0cc153209e414ca8168661f57827aa03ab84e7041ef9270ff639bcc519d24f5 # access the url @@ -87,7 +91,7 @@ $ curl "http://localhost:8333/newbucket/t.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X ### SSE-KMS ```bash -aws --endpoint-url http://localhost:8333 s3 cp file.txt s3://bucket/kms.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://bucket/kms.txt \ --sse aws:kms \ --sse-kms-key-id "test-key-123" ``` @@ -96,14 +100,14 @@ aws --endpoint-url http://localhost:8333 s3 cp file.txt s3://bucket/kms.txt \ ```bash # Generate a 256-bit key openssl rand -base64 32 > key.b64 -aws --endpoint-url http://localhost:8333 s3 cp file.txt s3://bucket/ssec.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://bucket/ssec.txt \ --sse-c AES256 \ --sse-c-key fileb://key.b64 ``` ### SSE-S3 (Server-managed) ```bash -aws --endpoint-url http://localhost:8333 s3 cp file.txt s3://bucket/sse-s3.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://bucket/sse-s3.txt \ --sse AES256 ``` diff --git a/Amazon-S3-API.md b/Amazon-S3-API.md index 51147e8..a4f185f 100644 --- a/Amazon-S3-API.md +++ b/Amazon-S3-API.md @@ -133,8 +133,11 @@ SeaweedFS supports S3 user-defined metadata via `x-amz-meta-*` headers. This all ## Setting User Metadata ```bash +# Set environment variables +export S3_ENDPOINT=http://localhost:8333 + # Using AWS CLI -aws s3 cp myfile.txt s3://mybucket/myfile.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp myfile.txt s3://mybucket/myfile.txt \ --metadata "expire=2025-12-01,author=john,project=demo" # Using curl @@ -142,7 +145,7 @@ curl -X PUT \ -H "x-amz-meta-expire: 2025-12-01" \ -H "x-amz-meta-author: john" \ --data-binary @myfile.txt \ - "http://localhost:8333/mybucket/myfile.txt" + "$S3_ENDPOINT/mybucket/myfile.txt" ``` ## Reading User Metadata @@ -151,10 +154,10 @@ User metadata is returned in response headers when you GET or HEAD an object: ```bash # Using AWS CLI -aws s3api head-object --bucket mybucket --key myfile.txt +aws --endpoint-url $S3_ENDPOINT s3api head-object --bucket mybucket --key myfile.txt # Using curl -curl -I "http://localhost:8333/mybucket/myfile.txt" +curl -I "$S3_ENDPOINT/mybucket/myfile.txt" # Response includes: # x-amz-meta-expire: 2025-12-01 # x-amz-meta-author: john @@ -165,7 +168,7 @@ curl -I "http://localhost:8333/mybucket/myfile.txt" To update metadata, use CopyObject with `x-amz-metadata-directive: REPLACE`: ```bash -aws s3 cp s3://mybucket/myfile.txt s3://mybucket/myfile.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp s3://mybucket/myfile.txt s3://mybucket/myfile.txt \ --metadata "expire=2026-01-01" \ --metadata-directive REPLACE ``` @@ -223,13 +226,13 @@ For detailed setup guides and examples, see: ```bash # SSE-KMS (Key Management Service) -aws s3 cp file.txt s3://mybucket/kms-encrypted.txt --server-side-encryption aws:kms --ssekms-key-id alias/my-key +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/kms-encrypted.txt --server-side-encryption aws:kms --ssekms-key-id alias/my-key # SSE-C (Customer-provided keys) -aws s3 cp file.txt s3://mybucket/customer-encrypted.txt --sse-c AES256 --sse-c-key fileb://my-key.bin +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/customer-encrypted.txt --sse-c AES256 --sse-c-key fileb://my-key.bin # SSE-S3 (Server-managed) -aws s3 cp file.txt s3://mybucket/server-encrypted.txt --server-side-encryption AES256 +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/server-encrypted.txt --server-side-encryption AES256 ``` # S3 Conditional Operations @@ -253,17 +256,17 @@ For detailed usage patterns and examples, see **[S3 Conditional Operations](S3-C ```bash # Get current ETag -ETAG=$(aws s3api head-object --bucket mybucket --key file.txt --query ETag --output text) +ETAG=$(aws --endpoint-url $S3_ENDPOINT s3api head-object --bucket mybucket --key file.txt --query ETag --output text) # Conditional update (optimistic locking) -curl -X PUT -H "If-Match: $ETAG" -d "updated content" "http://localhost:8333/mybucket/file.txt" +curl -X PUT -H "If-Match: $ETAG" -d "updated content" "$S3_ENDPOINT/mybucket/file.txt" # Conditional download (caching) -curl -H "If-None-Match: $ETAG" "http://localhost:8333/mybucket/file.txt" +curl -H "If-None-Match: $ETAG" "$S3_ENDPOINT/mybucket/file.txt" # Returns 304 Not Modified if unchanged # Prevent overwrite (atomic create) -curl -X PUT -H "If-None-Match: *" -d "new content" "http://localhost:8333/mybucket/newfile.txt" +curl -X PUT -H "If-None-Match: *" -d "new content" "$S3_ENDPOINT/mybucket/newfile.txt" ``` # S3 Authentication diff --git a/Quick-Start-with-weed-mini.md b/Quick-Start-with-weed-mini.md index c3a6b3b..4f97aad 100644 --- a/Quick-Start-with-weed-mini.md +++ b/Quick-Start-with-weed-mini.md @@ -67,6 +67,7 @@ Set AWS credentials before starting: ```bash export AWS_ACCESS_KEY_ID=your-access-key export AWS_SECRET_ACCESS_KEY=your-secret-key +export S3_ENDPOINT=http://localhost:8333 weed mini -dir=/data ``` @@ -91,8 +92,11 @@ By default, the SeaweedFS S3 gateway starts in **"Allow All" mode** if no S3 cre To enable authentication for `weed mini`, use one of the following methods: 1. **Environment Variables**: Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` before starting. ```bash - AWS_ACCESS_KEY_ID=any AWS_SECRET_ACCESS_KEY=any weed mini -dir=/data - ``` +export AWS_ACCESS_KEY_ID=any +export AWS_SECRET_ACCESS_KEY=any +export S3_ENDPOINT=http://localhost:8333 +weed mini -dir=/data +``` 2. **Config File**: Use the `-s3.config` flag with a JSON credentials file. ```bash weed mini -dir=/data -s3.config=s3.config @@ -109,10 +113,10 @@ If you haven't configured any credentials, you can use the S3 endpoint with the ```bash # Create a bucket -aws s3 --endpoint-url=http://localhost:8333 --no-sign-request mb s3://my-bucket +aws --endpoint-url $S3_ENDPOINT s3 mb s3://my-bucket --no-sign-request # List contents -aws s3 --endpoint-url=http://localhost:8333 --no-sign-request ls s3://my-bucket/ +aws --endpoint-url $S3_ENDPOINT s3 ls s3://my-bucket/ --no-sign-request ``` ### Authenticated Access @@ -126,7 +130,7 @@ aws configure # AWS Secret Access Key: # Use the local endpoint -aws s3 --endpoint-url=http://localhost:8333 ls s3://my-bucket/ +aws --endpoint-url $S3_ENDPOINT s3 ls s3://my-bucket/ ``` ## Web Interfaces diff --git a/S3-API-FAQ.md b/S3-API-FAQ.md index 19d0235..cbedc3a 100644 --- a/S3-API-FAQ.md +++ b/S3-API-FAQ.md @@ -64,11 +64,11 @@ fs.configure -locationPrefix /buckets/f341868e-baff-4e20-896a-08bc148e32f9/my-d ] } ``` -Save that in a `.json` file and call it, for example, via the `aws` cli: +Save that in a `.json` file and call it, for example, via the `aws` cli (ensure `$S3_ENDPOINT` is set, e.g., `export S3_ENDPOINT=http://localhost:8333`): -``` +```bash BUCKET_NAME=f341868e-baff-4e20-896a-08bc148e32f9 -aws --endpoint-url http://127.0.0.1:8333 s3api put-bucket-lifecycle-configuration --bucket $BUCKET_NAME --lifecycle-configuration "file://lifecycle_policy.json" +aws --endpoint-url $S3_ENDPOINT s3api put-bucket-lifecycle-configuration --bucket $BUCKET_NAME --lifecycle-configuration "file://lifecycle_policy.json" ``` Note that you don't need to add the part `/buckets/$BUCKET_NAME` in the configurations "Filter.Prefix" (contrary to using `fs.configure`, this is taken care of for you in the S3 API. @@ -209,10 +209,10 @@ You can test your reverse proxy configuration using AWS CLI: ```bash # Test basic bucket listing -aws s3 ls --endpoint-url https://yourdomain.com/s3 +aws --endpoint-url $S3_ENDPOINT s3 ls # Test presigned URL generation and access -aws s3 presign s3://test-bucket/test-object --endpoint-url https://yourdomain.com/s3 +aws --endpoint-url $S3_ENDPOINT s3 presign s3://test-bucket/test-object ``` For detailed configuration examples, see the [[S3-Nginx-Proxy]] documentation. diff --git a/S3-CORS.md b/S3-CORS.md index f2b1ee2..31b625e 100644 --- a/S3-CORS.md +++ b/S3-CORS.md @@ -42,7 +42,7 @@ This global configuration: SeaweedFS uses the following priority order: -1. **Bucket-level CORS** (if configured via `aws s3api put-bucket-cors`) - highest priority +1. **Bucket-level CORS** (if configured via `aws --endpoint-url $S3_ENDPOINT s3api put-bucket-cors`) - highest priority 2. **Global CORS** (from `-s3.allowedOrigins` parameter) - fallback if no bucket config 3. **No CORS** (if neither is configured) - no CORS headers applied @@ -85,7 +85,8 @@ A CORS configuration consists of one or more CORS rules. Each rule defines: Use the `PutBucketCors` API to set CORS configuration for a bucket: ```bash -aws s3api put-bucket-cors \ +# Ensure $S3_ENDPOINT is set (e.g., export S3_ENDPOINT=http://localhost:8333) +aws --endpoint-url $S3_ENDPOINT s3api put-bucket-cors \ --bucket my-bucket \ --cors-configuration file://cors-config.json ``` @@ -111,7 +112,7 @@ Example `cors-config.json`: Retrieve the current CORS configuration for a bucket: ```bash -aws s3api get-bucket-cors --bucket my-bucket +aws --endpoint-url $S3_ENDPOINT s3api get-bucket-cors --bucket my-bucket ``` Response: @@ -135,7 +136,7 @@ Response: Remove CORS configuration from a bucket: ```bash -aws s3api delete-bucket-cors --bucket my-bucket +aws --endpoint-url $S3_ENDPOINT s3api delete-bucket-cors --bucket my-bucket ``` ## CORS Rule Examples diff --git a/S3-Credentials.md b/S3-Credentials.md index eb97203..c4b6f27 100644 --- a/S3-Credentials.md +++ b/S3-Credentials.md @@ -79,6 +79,7 @@ Use AWS standard environment variables as a fallback when no other configuration ```bash export AWS_ACCESS_KEY_ID=your_access_key export AWS_SECRET_ACCESS_KEY=your_secret_key +export S3_ENDPOINT=http://localhost:8333 weed s3 -filer=localhost:8888 ``` @@ -317,7 +318,7 @@ Environment variables create global admin access and cannot be scoped to specifi **User cannot access intended bucket:** - Verify bucket name spelling in actions - Check that all required actions are granted (e.g., `List` for listing objects) -- Test with AWS CLI: `aws --endpoint-url=http://localhost:8333 s3 ls s3://mybucket` +- Test with AWS CLI: `aws --endpoint-url $S3_ENDPOINT s3 ls s3://mybucket` ## Anonymous Access @@ -455,7 +456,7 @@ weed shell Test credentials: ```bash # Test with AWS CLI -aws --endpoint-url=http://localhost:8333 s3 ls +aws --endpoint-url $S3_ENDPOINT s3 ls ``` ## Security Best Practices diff --git a/S3-Object-Lock-and-Retention.md b/S3-Object-Lock-and-Retention.md index 6af1ebb..f105984 100644 --- a/S3-Object-Lock-and-Retention.md +++ b/S3-Object-Lock-and-Retention.md @@ -12,6 +12,11 @@ Object Lock is a feature that allows you to store objects using a WORM (Write On - **Immutable Setting**: Object Lock can only be enabled when creating a bucket, not on existing buckets - **S3 API**: All Object Lock operations are available through the S3 API +### Configuration +```bash +export S3_ENDPOINT=http://localhost:8333 +``` + ## Key Features ### 1. Retention Modes @@ -63,12 +68,12 @@ Object Lock must be enabled when creating a bucket: ```bash # Create bucket with Object Lock enabled -aws s3api create-bucket \ +aws --endpoint-url $S3_ENDPOINT s3api create-bucket \ --bucket my-secure-bucket \ --object-lock-enabled-for-bucket # Enable versioning (automatically done with Object Lock) -aws s3api put-bucket-versioning \ +aws --endpoint-url $S3_ENDPOINT s3api put-bucket-versioning \ --bucket my-secure-bucket \ --versioning-configuration Status=Enabled ``` @@ -105,7 +110,7 @@ Grant users permission to bypass governance retention: #### Set Governance Retention ```bash # Set 30-day governance retention -aws s3api put-object-retention \ +aws --endpoint-url $S3_ENDPOINT s3api put-object-retention \ --bucket my-secure-bucket \ --key important-document.pdf \ --retention '{ @@ -117,7 +122,7 @@ aws s3api put-object-retention \ #### Set Compliance Retention ```bash # Set 7-year compliance retention -aws s3api put-object-retention \ +aws --endpoint-url $S3_ENDPOINT s3api put-object-retention \ --bucket my-secure-bucket \ --key regulatory-record.json \ --retention '{ @@ -128,7 +133,7 @@ aws s3api put-object-retention \ #### Get Object Retention ```bash -aws s3api get-object-retention \ +aws --endpoint-url $S3_ENDPOINT s3api get-object-retention \ --bucket my-secure-bucket \ --key important-document.pdf ``` @@ -137,7 +142,7 @@ aws s3api get-object-retention \ #### Apply Legal Hold ```bash -aws s3api put-object-legal-hold \ +aws --endpoint-url $S3_ENDPOINT s3api put-object-legal-hold \ --bucket my-secure-bucket \ --key investigation-file.doc \ --legal-hold Status=ON @@ -145,7 +150,7 @@ aws s3api put-object-legal-hold \ #### Remove Legal Hold ```bash -aws s3api put-object-legal-hold \ +aws --endpoint-url $S3_ENDPOINT s3api put-object-legal-hold \ --bucket my-secure-bucket \ --key investigation-file.doc \ --legal-hold Status=OFF @@ -153,7 +158,7 @@ aws s3api put-object-legal-hold \ #### Check Legal Hold Status ```bash -aws s3api get-object-legal-hold \ +aws --endpoint-url $S3_ENDPOINT s3api get-object-legal-hold \ --bucket my-secure-bucket \ --key investigation-file.doc ``` @@ -163,7 +168,7 @@ aws s3api get-object-legal-hold \ #### Delete Object with Governance Bypass ```bash # User with bypass permission can delete governance-protected objects -aws s3api delete-object \ +aws --endpoint-url $S3_ENDPOINT s3api delete-object \ --bucket my-secure-bucket \ --key document.pdf \ --bypass-governance-retention @@ -172,7 +177,7 @@ aws s3api delete-object \ #### Bulk Delete with Governance Bypass ```bash # Delete multiple objects with governance bypass -aws s3api delete-objects \ +aws --endpoint-url $S3_ENDPOINT s3api delete-objects \ --bucket my-secure-bucket \ --delete file://delete-objects.json \ --bypass-governance-retention @@ -182,14 +187,14 @@ aws s3api delete-objects \ #### List Object Versions ```bash -aws s3api list-object-versions \ +aws --endpoint-url $S3_ENDPOINT s3api list-object-versions \ --bucket my-secure-bucket \ --prefix documents/ ``` #### Get Specific Version ```bash -aws s3api get-object \ +aws --endpoint-url $S3_ENDPOINT s3api get-object \ --bucket my-secure-bucket \ --key document.pdf \ --version-id "3/L4kqtJlcpXroDTDmpUMLUo" @@ -197,7 +202,7 @@ aws s3api get-object \ #### Delete Specific Version ```bash -aws s3api delete-object \ +aws --endpoint-url $S3_ENDPOINT s3api delete-object \ --bucket my-secure-bucket \ --key document.pdf \ --version-id "3/L4kqtJlcpXroDTDmpUMLUo" @@ -371,16 +376,16 @@ SeaweedFS Object Lock is fully compatible with AWS S3 Object Lock: ```bash # Check bucket Object Lock configuration -aws s3api get-object-lock-configuration --bucket my-secure-bucket +aws --endpoint-url $S3_ENDPOINT s3api get-object-lock-configuration --bucket my-secure-bucket # Check bucket versioning status -aws s3api get-bucket-versioning --bucket my-secure-bucket +aws --endpoint-url $S3_ENDPOINT s3api get-bucket-versioning --bucket my-secure-bucket # Check object retention -aws s3api get-object-retention --bucket my-secure-bucket --key file.pdf +aws --endpoint-url $S3_ENDPOINT s3api get-object-retention --bucket my-secure-bucket --key file.pdf # Check legal hold status -aws s3api get-object-legal-hold --bucket my-secure-bucket --key file.pdf +aws --endpoint-url $S3_ENDPOINT s3api get-object-legal-hold --bucket my-secure-bucket --key file.pdf ``` ## Migration Guide @@ -388,7 +393,7 @@ aws s3api get-object-legal-hold --bucket my-secure-bucket --key file.pdf ### From Non-Object Lock Buckets 1. **Create New Bucket**: Create new bucket with Object Lock enabled -2. **Copy Data**: Use `aws s3 sync` to copy data to new bucket +2. **Copy Data**: Use `aws --endpoint-url $S3_ENDPOINT s3 sync` to copy data to new bucket 3. **Apply Retention**: Set retention policies on copied objects 4. **Update Applications**: Update applications to use new bucket 5. **Cleanup**: Remove old bucket after validation @@ -399,10 +404,10 @@ Object Lock configurations can be migrated using standard S3 API calls: ```bash # Export current configuration -aws s3api get-object-lock-configuration --bucket source-bucket > config.json +aws --endpoint-url $S3_ENDPOINT s3api get-object-lock-configuration --bucket source-bucket > config.json # Apply to SeaweedFS bucket -aws s3api put-object-lock-configuration \ +aws --endpoint-url $S3_ENDPOINT s3api put-object-lock-configuration \ --bucket target-bucket \ --object-lock-configuration file://config.json ``` diff --git a/S3-Object-Versioning.md b/S3-Object-Versioning.md index 24780b4..460b68a 100644 --- a/S3-Object-Versioning.md +++ b/S3-Object-Versioning.md @@ -2,12 +2,15 @@ SeaweedFS supports S3 object versioning, which allows you to keep multiple variants of an object in the same bucket. This provides data protection against accidental deletion or modification. -## Enable Versioning +### Configuration +```bash +export S3_ENDPOINT=http://localhost:8333 +``` To enable versioning on a bucket, use the `PutBucketVersioning` API: ```bash -aws s3api put-bucket-versioning \ +aws --endpoint-url $S3_ENDPOINT s3api put-bucket-versioning \ --bucket my-bucket \ --versioning-configuration Status=Enabled ``` @@ -17,7 +20,7 @@ aws s3api put-bucket-versioning \ To check the versioning status of a bucket: ```bash -aws s3api get-bucket-versioning --bucket my-bucket +aws --endpoint-url $S3_ENDPOINT s3api get-bucket-versioning --bucket my-bucket ``` Response: @@ -32,7 +35,7 @@ Response: To suspend versioning (not disable completely): ```bash -aws s3api put-bucket-versioning \ +aws --endpoint-url $S3_ENDPOINT s3api put-bucket-versioning \ --bucket my-bucket \ --versioning-configuration Status=Suspended ``` @@ -42,7 +45,7 @@ aws s3api put-bucket-versioning \ To list all versions of objects in a bucket: ```bash -aws s3api list-object-versions --bucket my-bucket +aws --endpoint-url $S3_ENDPOINT s3api list-object-versions --bucket my-bucket ``` Response includes both object versions and delete markers: @@ -73,7 +76,7 @@ Response includes both object versions and delete markers: ### Get a specific version of an object: ```bash -aws s3api get-object \ +aws --endpoint-url $S3_ENDPOINT s3api get-object \ --bucket my-bucket \ --key example.txt \ --version-id v_1234567890abcdef \ @@ -82,7 +85,7 @@ aws s3api get-object \ ### Copy a specific version: ```bash -aws s3api copy-object \ +aws --endpoint-url $S3_ENDPOINT s3api copy-object \ --copy-source my-bucket/example.txt?versionId=v_1234567890abcdef \ --bucket my-bucket \ --key example-copy.txt @@ -90,7 +93,7 @@ aws s3api copy-object \ ### Delete a specific version: ```bash -aws s3api delete-object \ +aws --endpoint-url $S3_ENDPOINT s3api delete-object \ --bucket my-bucket \ --key example.txt \ --version-id v_1234567890abcdef diff --git a/S3-Policy-Variables.md b/S3-Policy-Variables.md index e39dcf9..3fbf854 100644 --- a/S3-Policy-Variables.md +++ b/S3-Policy-Variables.md @@ -284,17 +284,17 @@ Combine Allow and Deny statements to prevent unauthorized access: ### 2. Test Policies Thoroughly -Test with different users to ensure variables substitute correctly: +Test with different users to ensure variables substitute correctly (ensure `$S3_ENDPOINT` is set, e.g., `export S3_ENDPOINT=http://localhost:8333`): ```bash # Test as alice -aws s3 cp file.txt s3://bucket/alice/file.txt --profile alice +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://bucket/alice/file.txt --profile alice # Test as bob -aws s3 cp file.txt s3://bucket/bob/file.txt --profile bob +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://bucket/bob/file.txt --profile bob # Verify isolation -aws s3 ls s3://bucket/alice/ --profile bob # Should fail +aws --endpoint-url $S3_ENDPOINT s3 ls s3://bucket/alice/ --profile bob # Should fail ``` ### 3. Use Conditions for Complex Logic diff --git a/Server-Side-Encryption-SSE-C.md b/Server-Side-Encryption-SSE-C.md index 60f2fd1..2b48372 100644 --- a/Server-Side-Encryption-SSE-C.md +++ b/Server-Side-Encryption-SSE-C.md @@ -25,12 +25,15 @@ X-Amz-Server-Side-Encryption-Customer-Key-MD5: ### Upload Encrypted Object ```bash +# Set endpoint environment variable +export S3_ENDPOINT=http://localhost:8333 + # Generate a 256-bit key KEY=$(openssl rand -base64 32) KEY_MD5=$(echo -n "$KEY" | base64 -d | md5sum | cut -d' ' -f1) # Upload encrypted object -curl -X PUT "http://localhost:8333/bucket/encrypted-file.txt" \ +curl -X PUT "$S3_ENDPOINT/bucket/encrypted-file.txt" \ -H "X-Amz-Server-Side-Encryption-Customer-Algorithm: AES256" \ -H "X-Amz-Server-Side-Encryption-Customer-Key: $KEY" \ -H "X-Amz-Server-Side-Encryption-Customer-Key-MD5: $KEY_MD5" \ @@ -42,7 +45,7 @@ curl -X PUT "http://localhost:8333/bucket/encrypted-file.txt" \ ```bash # Download and decrypt object (must use same key) -curl "http://localhost:8333/bucket/encrypted-file.txt" \ +curl "$S3_ENDPOINT/bucket/encrypted-file.txt" \ -H "X-Amz-Server-Side-Encryption-Customer-Algorithm: AES256" \ -H "X-Amz-Server-Side-Encryption-Customer-Key: $KEY" \ -H "X-Amz-Server-Side-Encryption-Customer-Key-MD5: $KEY_MD5" @@ -52,7 +55,7 @@ curl "http://localhost:8333/bucket/encrypted-file.txt" \ ```bash # Get metadata for encrypted object -curl -I "http://localhost:8333/bucket/encrypted-file.txt" \ +curl -I "$S3_ENDPOINT/bucket/encrypted-file.txt" \ -H "X-Amz-Server-Side-Encryption-Customer-Algorithm: AES256" \ -H "X-Amz-Server-Side-Encryption-Customer-Key: $KEY" \ -H "X-Amz-Server-Side-Encryption-Customer-Key-MD5: $KEY_MD5" @@ -62,7 +65,7 @@ curl -I "http://localhost:8333/bucket/encrypted-file.txt" \ ```bash # Copy encrypted object to new location (same key) -curl -X PUT "http://localhost:8333/bucket/copied-file.txt" \ +curl -X PUT "$S3_ENDPOINT/bucket/copied-file.txt" \ -H "x-amz-copy-source: /bucket/encrypted-file.txt" \ -H "X-Amz-Server-Side-Encryption-Customer-Algorithm: AES256" \ -H "X-Amz-Server-Side-Encryption-Customer-Key: $KEY" \ @@ -75,7 +78,7 @@ curl -X PUT "http://localhost:8333/bucket/copied-file.txt" \ NEW_KEY=$(openssl rand -base64 32) NEW_KEY_MD5=$(echo -n "$NEW_KEY" | base64 -d | md5sum | cut -d' ' -f1) -curl -X PUT "http://localhost:8333/bucket/reencrypted-file.txt" \ +curl -X PUT "$S3_ENDPOINT/bucket/reencrypted-file.txt" \ -H "x-amz-copy-source: /bucket/encrypted-file.txt" \ -H "X-Amz-Server-Side-Encryption-Customer-Algorithm: AES256" \ -H "X-Amz-Server-Side-Encryption-Customer-Key: $NEW_KEY" \ @@ -89,17 +92,17 @@ curl -X PUT "http://localhost:8333/bucket/reencrypted-file.txt" \ ```bash # Upload with SSE-C -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --sse-c AES256 \ --sse-c-key fileb://customer-key.bin # Download with SSE-C -aws s3 cp s3://mybucket/file.txt downloaded-file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp s3://mybucket/file.txt downloaded-file.txt \ --sse-c AES256 \ --sse-c-key fileb://customer-key.bin # Copy with SSE-C (same key) -aws s3 cp s3://mybucket/file.txt s3://mybucket/file-copy.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp s3://mybucket/file.txt s3://mybucket/file-copy.txt \ --sse-c AES256 \ --sse-c-key fileb://customer-key.bin \ --sse-c-copy-source AES256 \ @@ -128,7 +131,7 @@ X-Amz-Server-Side-Encryption-Customer-Key: dGVzdA== # Error! (too short) **Missing key for encrypted object:** ```bash -curl http://localhost:8333/bucket/encrypted-file.txt # Error! +curl "$S3_ENDPOINT/bucket/encrypted-file.txt" # Error! ``` ## Related Documentation diff --git a/Server-Side-Encryption-SSE-KMS.md b/Server-Side-Encryption-SSE-KMS.md index 633c4a9..90e604e 100644 --- a/Server-Side-Encryption-SSE-KMS.md +++ b/Server-Side-Encryption-SSE-KMS.md @@ -54,8 +54,11 @@ weed s3 -config=s3_kms_config.json -port=8333 ### 3. Test the Integration ```bash +# Ensure $S3_ENDPOINT is set +export S3_ENDPOINT=http://localhost:8333 + # Upload object with SSE-KMS -aws s3 cp test-file.txt s3://mybucket/test-file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp test-file.txt s3://mybucket/test-file.txt \ --server-side-encryption aws:kms \ --ssekms-key-id alias/my-key ``` @@ -104,7 +107,7 @@ aws kms create-alias \ ```bash # Upload with AWS KMS encryption -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --server-side-encryption aws:kms \ --ssekms-key-id alias/seaweedfs-key ``` @@ -201,7 +204,7 @@ openbao write -f transit/keys/seaweedfs-key ```bash # Upload with Vault encryption -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --server-side-encryption aws:kms \ --ssekms-key-id seaweedfs-key ``` diff --git a/Server-Side-Encryption.md b/Server-Side-Encryption.md index 316abd5..b56999f 100644 --- a/Server-Side-Encryption.md +++ b/Server-Side-Encryption.md @@ -38,9 +38,12 @@ Use this quick guide to choose the right option: ### SSE-KMS (Enterprise) ```bash +# Ensure $S3_ENDPOINT is set +export S3_ENDPOINT=http://localhost:8333 + # Configure KMS in s3 config file (see KMS Providers Integration guide) # Then upload with KMS encryption -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --server-side-encryption aws:kms \ --ssekms-key-id test-key-123 ``` @@ -51,7 +54,7 @@ aws s3 cp file.txt s3://mybucket/file.txt \ openssl rand 32 > customer-key.bin # Upload with customer-provided key -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --sse-c AES256 \ --sse-c-key fileb://customer-key.bin ``` @@ -59,7 +62,7 @@ aws s3 cp file.txt s3://mybucket/file.txt \ ### SSE-S3 (Server-Managed) ```bash # Explicit SSE-S3 on upload (or configure bucket default encryption) -aws s3 cp file.txt s3://mybucket/file.txt \ +aws --endpoint-url $S3_ENDPOINT s3 cp file.txt s3://mybucket/file.txt \ --server-side-encryption AES256 ```