mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-21 19:57:39 +01:00
Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e604b19d9d | ||
|
|
428af1c0eb | ||
|
|
fa872adb7f | ||
|
|
b95b7f5761 | ||
|
|
19d7c0224b | ||
|
|
a5ac780e66 | ||
|
|
291353b1e3 | ||
|
|
84e227dbc3 | ||
|
|
e144503caf | ||
|
|
327495006a | ||
|
|
0b5d1c0c64 | ||
|
|
56468c83e4 | ||
|
|
bfe4b810bf | ||
|
|
618febdba5 | ||
|
|
ed9d1eec64 | ||
|
|
f07aabb39f | ||
|
|
17fe96e620 | ||
|
|
35e9f84334 | ||
|
|
10b64686ba | ||
|
|
045c834dcf | ||
|
|
318e1c64d6 | ||
|
|
2c84bb1161 | ||
|
|
61348b147b | ||
|
|
a7fec8004e | ||
|
|
b65bcd4afa | ||
|
|
79297b549e | ||
|
|
4c7e5afbfe | ||
|
|
9cb7dc7204 | ||
|
|
77a2b1b378 | ||
|
|
7387866fd6 | ||
|
|
4bf126944f | ||
|
|
1737211ffa | ||
|
|
4ee57f214c | ||
|
|
211bf4d2fe | ||
|
|
deb8b9bef1 | ||
|
|
53fe128511 | ||
|
|
6863412f4e |
No files matched your search
@@ -9,7 +9,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
push:
|
||||
branches: [ master, main ]
|
||||
@@ -19,7 +18,6 @@ on:
|
||||
- 'weed/storage/**'
|
||||
- 'weed/pb/volume_server.proto'
|
||||
- 'weed/pb/volume_server_pb/**'
|
||||
- 'rust/volume_server/**'
|
||||
- '.github/workflows/volume-server-integration-tests.yml'
|
||||
|
||||
concurrency:
|
||||
@@ -143,77 +141,3 @@ jobs:
|
||||
echo "## Volume Server Integration Test Summary (${{ matrix.test-type }} - Shard ${{ matrix.shard }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Suite: test/volume_server/${{ matrix.test-type }} (shard ${{ matrix.shard }} of ${SHARD_COUNT}, see 'Select this shard's tests' for the split)" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Command: go test -v -count=1 -timeout=${{ env.TEST_TIMEOUT }} ./test/volume_server/${{ matrix.test-type }}/... -run \"\${TEST_PATTERN}\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
volume-server-rust-smoke:
|
||||
name: Volume Server Rust Smoke (${{ matrix.mode }})
|
||||
runs-on: ubuntu-22.04
|
||||
timeout-minutes: 40
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
mode: [exec, proxy, native]
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: 'go.mod'
|
||||
|
||||
- name: Set up Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- name: Build SeaweedFS binary
|
||||
run: |
|
||||
cd weed
|
||||
go build -o weed .
|
||||
chmod +x weed
|
||||
./weed version
|
||||
|
||||
- name: Build Rust volume launcher
|
||||
run: |
|
||||
cd rust/volume_server
|
||||
cargo build --release
|
||||
|
||||
- name: Run Rust-mode HTTP smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run "TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip"
|
||||
|
||||
- name: Run Rust-mode gRPC smoke tests
|
||||
env:
|
||||
WEED_BINARY: ${{ github.workspace }}/weed/weed
|
||||
VOLUME_SERVER_IMPL: rust
|
||||
RUST_VOLUME_BINARY: ${{ github.workspace }}/rust/volume_server/target/release/weed-volume-rs
|
||||
VOLUME_SERVER_RUST_MODE: ${{ matrix.mode }}
|
||||
run: |
|
||||
go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run "TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus"
|
||||
|
||||
- name: Collect logs on failure
|
||||
if: failure()
|
||||
run: |
|
||||
mkdir -p /tmp/volume-server-rust-smoke-logs
|
||||
find /tmp -maxdepth 1 -type d -name "seaweedfs_volume_server_it_*" -print -exec cp -r {} /tmp/volume-server-rust-smoke-logs/ \; || true
|
||||
|
||||
- name: Archive logs on failure
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: volume-server-rust-smoke-logs-${{ matrix.mode }}
|
||||
path: /tmp/volume-server-rust-smoke-logs/
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Rust smoke summary
|
||||
if: always()
|
||||
run: |
|
||||
echo "## Volume Server Rust Smoke Summary (${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- Mode: Go master + Rust volume server launcher (VOLUME_SERVER_IMPL=rust, VOLUME_SERVER_RUST_MODE=${{ matrix.mode }})" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- HTTP Command: go test -v -count=1 -timeout=25m ./test/volume_server/http/... -run \"TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "- gRPC Command: go test -v -count=1 -timeout=25m ./test/volume_server/grpc/... -run \"TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus\"" >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -1,6 +1,6 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "4.50"
|
||||
appVersion: "4.49"
|
||||
# Dev note: Trigger a helm chart release by `git tag -a helm-<version>`
|
||||
version: 4.50.0
|
||||
version: 4.49.0
|
||||
@@ -170,6 +170,9 @@ spec:
|
||||
{{- if .Values.filer.metricsIp }}
|
||||
-metricsIp={{ .Values.filer.metricsIp }} \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.redirectOnRead }}
|
||||
-redirectOnRead \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.disableHttp }}
|
||||
-disableHttp \
|
||||
{{- end }}
|
||||
|
||||
@@ -740,6 +740,9 @@ filer:
|
||||
# encrypt data on volume servers
|
||||
encryptVolumeData: false
|
||||
|
||||
# Whether proxy or redirect to volume server during file GET request
|
||||
redirectOnRead: false
|
||||
|
||||
# Limit sub dir listing size (default 100000)
|
||||
dirListLimit: 100000
|
||||
|
||||
|
||||
+1080
-1034
File diff suppressed because it is too large.
Load diff
|
Before Width: | Height: | Size: 53 KiB After Width: | Height: | Size: 55 KiB |
Generated
-7
@@ -1,7 +0,0 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
@@ -1,6 +0,0 @@
|
||||
[package]
|
||||
name = "weed-volume-rs"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
@@ -1,245 +0,0 @@
|
||||
# Rust Volume Server Parity Implementation Plan
|
||||
|
||||
## Objective
|
||||
Implement a native Rust volume server that replicates Go volume-server behavior for HTTP and gRPC APIs, so it can become a drop-in replacement validated by existing integration suites.
|
||||
|
||||
## Current Focus (2026-02-16)
|
||||
- Program focus is now Rust implementation parity, not broad test expansion.
|
||||
- `test/volume_server` is treated as the parity gate.
|
||||
- Existing Rust launcher modes (`exec`, `proxy`) are transition tools; they are not the final target.
|
||||
|
||||
## Current Status
|
||||
- Rust crate and launcher are in place.
|
||||
- Integration harness can run:
|
||||
- Go master + Go volume (default)
|
||||
- Go master + Rust launcher (`VOLUME_SERVER_IMPL=rust`)
|
||||
- Rust launcher `proxy` mode has full-suite integration pass while delegating backend handlers to Go.
|
||||
- Rust launcher `native` mode is wired as the default Rust entrypoint and currently bootstraps via supervised Go backend delegation.
|
||||
- Native Rust HTTP handlers now serve control/surface paths in `native` mode:
|
||||
- `/status`, `/healthz`
|
||||
- `OPTIONS` admin/public behavior (method allow-list + CORS preflight headers)
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*`
|
||||
- public non-read methods (`POST`/`PUT`/`DELETE`/unsupported verbs) as `200` no-op passthrough parity
|
||||
- admin unsupported verbs as `400` parity
|
||||
- Native Rust control-surface parity now also includes:
|
||||
- `/healthz` status mirroring from backend state transitions (e.g. leave/stopping => `503`)
|
||||
- absolute-form HTTP request-target normalization before native route matching
|
||||
- Native Rust HTTP data-path prevalidation now includes:
|
||||
- early malformed vid/fid rejection (`400`) for GET/HEAD/POST/PUT fid-route shapes before delegation
|
||||
- slash-form fid-route parsing for `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` with reserved-path exclusions (`/status`, `/healthz`, `/ui/index.html`, `/stats/*`, static assets)
|
||||
- write-path error prevalidation for fid routes:
|
||||
- malformed multipart form-data requests without a boundary => `400`
|
||||
- requests containing `Content-MD5` header => `400` mismatch parity branch
|
||||
- request bodies exceeding configured `-fileSizeLimitMB` (by `Content-Length`) => `400`
|
||||
- Native Rust API/storage handlers are not implemented yet.
|
||||
|
||||
## Parity Exit Criteria
|
||||
1. Native mode passes:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
2. CI runs native Rust mode integration coverage (at least smoke, then expanded shards).
|
||||
3. Rust mode defaults to native behavior for integration harness.
|
||||
4. Go-backend delegation is removed (or retained only as explicit fallback mode).
|
||||
|
||||
## Architecture Workstreams
|
||||
|
||||
### A. Runtime and Configuration Parity
|
||||
- [x] Add `native` runtime mode in `weed-volume-rs` (bootstrap delegation path).
|
||||
- [ ] Parse and honor volume-server CLI/config flags used by integration harness:
|
||||
- [ ] network/bind ports (`-ip`, `-port`, `-port.grpc`, `-port.public`)
|
||||
- [ ] master target/config dir/read mode/throttling/JWT-related config
|
||||
- [ ] size/timeout controls and maintenance state defaults
|
||||
- [ ] Implement graceful lifecycle behavior (signals, shutdown, readiness).
|
||||
|
||||
### B. Native HTTP Surface
|
||||
- [ ] Admin/control endpoints:
|
||||
- [x] `GET /status` (native Rust in `native` mode)
|
||||
- [x] `GET /healthz` (native Rust in `native` mode)
|
||||
- [x] `OPTIONS` admin/public method+CORS control behavior (native Rust in `native` mode)
|
||||
- [x] static/UI endpoints currently exercised
|
||||
- [ ] Data read path parity:
|
||||
- [ ] fid parsing/path variants
|
||||
- [ ] conditional headers (`If-Modified-Since`, `If-None-Match`)
|
||||
- [ ] range handling (single/multi/invalid)
|
||||
- [ ] deleted reads, auth checks, read-mode branches
|
||||
- [ ] chunk-manifest and compression/image transformation branches
|
||||
- [ ] Data write/delete parity:
|
||||
- [ ] write success/unchanged/error paths
|
||||
- [ ] replication and file-size-limit paths
|
||||
- [ ] delete and chunk-manifest delete branches
|
||||
- [ ] Method/CORS/public-port parity for split admin/public behavior.
|
||||
|
||||
### C. Native gRPC Surface
|
||||
- [ ] Control-plane RPCs:
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- [ ] admin lifecycle: allocate/mount/unmount/delete/configure/readonly/writable
|
||||
- [ ] Data RPCs:
|
||||
- [ ] `ReadNeedleBlob`, `ReadNeedleMeta`, `WriteNeedleBlob`
|
||||
- [ ] `BatchDelete`, `ReadAllNeedles`
|
||||
- [ ] sync/copy/receive and status endpoints
|
||||
- [ ] Stream RPCs:
|
||||
- [ ] tail sender/receiver
|
||||
- [ ] vacuum streams
|
||||
- [ ] query streams
|
||||
- [ ] Advanced families:
|
||||
- [ ] erasure coding RPC set
|
||||
- [ ] tiering/remote fetch
|
||||
- [ ] scrub/query mode matrix
|
||||
|
||||
### D. Storage Compatibility Layer
|
||||
- [ ] Implement volume data/index handling compatible with Go on-disk format.
|
||||
- [ ] Preserve cookie/checksum/timestamp semantics used by tests.
|
||||
- [ ] Match read/write/delete consistency and error mapping behavior.
|
||||
- [ ] Ensure EC metadata/data-path compatibility with existing files.
|
||||
|
||||
### E. Operational Hardening
|
||||
- [ ] Deterministic startup/readiness and shutdown semantics.
|
||||
- [ ] Log/error parity sufficient for debugging and CI triage.
|
||||
- [ ] Concurrency/timeout behavior alignment for throttling and streams.
|
||||
- [ ] Performance baseline checks vs Go for key flows.
|
||||
|
||||
## Milestone Plan
|
||||
|
||||
### M0 (Completed): Harness + Launcher Transition
|
||||
- [x] Rust launcher integrated into harness.
|
||||
- [x] Proxy mode full-suite validation with Go backend delegation.
|
||||
|
||||
### M1: Native Skeleton (Control Plane First)
|
||||
- [ ] `native` mode boots and serves:
|
||||
- [x] `/status`, `/healthz`
|
||||
- [ ] `GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`
|
||||
- Gate:
|
||||
- [x] targeted HTTP/grpc control tests pass in `native` mode (delegated backend path).
|
||||
|
||||
### M2: Native Core Data Paths
|
||||
- [ ] Native HTTP read/write/delete baseline parity.
|
||||
- [ ] Native gRPC data baseline parity (`Read/WriteNeedle*`, `BatchDelete`, `ReadAllNeedles`).
|
||||
- Gate:
|
||||
- core HTTP and gRPC data suites pass in `native` mode.
|
||||
|
||||
### M3: Native Stream + Copy/Sync
|
||||
- [ ] Tail/copy/receive/sync paths in native mode.
|
||||
- Gate:
|
||||
- stream/copy families pass in `native` mode.
|
||||
|
||||
### M4: Native Advanced Feature Families
|
||||
- [ ] EC, tiering, scrub/query advanced branches.
|
||||
- Gate:
|
||||
- full `/test/volume_server/http` and `/test/volume_server/grpc` pass in `native` mode.
|
||||
|
||||
### M5: CI/Cutover
|
||||
- [x] Add/expand native-mode CI jobs (smoke matrix includes `native`).
|
||||
- [x] Make native mode default for Rust integration runs.
|
||||
- [ ] Keep `exec`/`proxy` only as explicit fallback modes during rollout.
|
||||
|
||||
## Immediate Next Steps
|
||||
1. Implement minimal native gRPC state/ping RPC handlers (`GetState`, `SetState`, `VolumeServerStatus`, `Ping`, `VolumeServerLeave`).
|
||||
2. Expand native HTTP parity from control/static surface into data-path handlers (read/write/delete variants) while preserving current native control behavior.
|
||||
3. Keep rerunning native-mode integration suites as each delegated branch is replaced.
|
||||
4. Add mismatch triage notes for each API moved from delegation to native implementation.
|
||||
|
||||
## Risk Register
|
||||
- On-disk format mismatch risk:
|
||||
- Mitigation: implement format-level compatibility tests early (idx/dat/needle encoding).
|
||||
- Behavioral drift in edge branches:
|
||||
- Mitigation: use integration suite failures as primary truth; only add tests for newly discovered untracked branches.
|
||||
- Stream/concurrency semantic mismatch:
|
||||
- Mitigation: stabilize with focused interruption/timeout parity tests.
|
||||
|
||||
## Progress Log
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust launcher integration (`exec`) and harness wiring.
|
||||
- Validation: Rust launcher mode passed smoke and full integration suites while delegating to Go backend.
|
||||
- Commits: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`, `3bd20e6a1`, `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy supervisor mode and validated full integration suite.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=proxy go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `a7f50d23b`, `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Re-focused plan from test expansion to native Rust implementation parity.
|
||||
- Validation basis: latest Rust proxy full-suite pass keeps regression baseline stable while native implementation starts.
|
||||
- Commits: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust launcher mode bootstrap, set Rust launcher default mode to `native`, and expanded CI Rust smoke matrix to include `native`.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust go test -count=1 ./test/volume_server/grpc -run '^TestStateAndStatusRPCs$'`
|
||||
- Commits: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP handlers in `native` mode for `/status` and `/healthz` on the admin listener while preserving proxy delegation for other APIs.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust `OPTIONS` handling for admin and public listeners in `native` mode, including method allow-list and origin-driven CORS response parity used by integration tests.
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust UI/static/public-surface handlers in `native` mode:
|
||||
- `/ui/index.html` with config-driven JWT/access-ui gating parity
|
||||
- `/favicon.ico` and `/seaweedfsstatic/*` static endpoint parity
|
||||
- public non-read methods as `200` no-op passthrough parity
|
||||
- admin unsupported methods as `400` parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestUiIndexNotExposedWhenJwtSigningEnabled$|^TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue$|^TestStaticAssetEndpoints$|^TestStaticAssetEndpointsOnPublicPort$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestOptionsMethodsByPort$|^TestOptionsWithOriginIncludesCorsHeaders$|^TestPublicPortReadOnlyMethodBehavior$|^TestCorsAndUnsupportedMethodBehavior$|^TestUnsupportedMethodTraceParity$|^TestUnsupportedMethodPropfindParity$|^TestUnsupportedMethodConnectParity$|^TestUnsupportedMethodMkcolParity$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc`
|
||||
- Commits: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Hardened native Rust HTTP control-path parity:
|
||||
- `/healthz` now mirrors backend health status (`200`/`503`) in native mode
|
||||
- absolute-form request-target parsing (e.g. `GET http://host/path HTTP/1.1`) now normalizes to route path parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native malformed fid-route validation in `native` mode:
|
||||
- GET/HEAD/POST/PUT requests matching fid URL shapes now return Rust-native `400 Bad Request` for invalid vid/fid tokens
|
||||
- valid fid-route requests still delegate to backend handlers for storage/data execution paths
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestInvalidReadPathReturnsBadRequest$|^TestWriteInvalidVidAndFidReturnBadRequest$|^TestMalformedVidFidPathReturnsBadRequest$|^TestDownloadLimitInvalidVidWhileOverLimitReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added native Rust write-path prevalidation branches for fid routes in `native` mode:
|
||||
- multipart form uploads without boundary now return native `400`
|
||||
- `Content-MD5` requests now return native `400` mismatch branch
|
||||
- `Content-Length` over configured `-fileSizeLimitMB` now returns native `400` limit branch
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestUploadReadRangeHeadDeleteRoundTrip$|^TestWriteMalformedMultipartAndMD5Mismatch$|^TestWriteRejectsPayloadOverFileSizeLimit$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened native fid-route parser to cover slash-form URLs:
|
||||
- `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` now flow through native malformed vid/fid validation
|
||||
- added explicit non-fid exclusions for control/stats/static paths to preserve route parity
|
||||
- Validation:
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http -run '^TestAdminStatusAndHealthz$|^TestReadPathShapesAndIfModifiedSince$|^TestMalformedVidFidPathReturnsBadRequest$'`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/http/...`
|
||||
- `env VOLUME_SERVER_IMPL=rust VOLUME_SERVER_RUST_MODE=native go test -count=1 ./test/volume_server/grpc/...`
|
||||
- Commits: `e66983e9b`
|
||||
File diff suppressed because it is too large.
Load diff
@@ -3039,62 +3039,50 @@ pub async fn post_handler(
|
||||
let needle_etag = n.etag();
|
||||
let needle_has_name = n.has_name();
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
let needs_replication = !is_replicate && {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
|
||||
// The fan-out runs concurrently with the local needle write, like Go's
|
||||
// ReplicatedWrite: it reads the already-buffered request body, so the two
|
||||
// sides share no mutable state. The ack still waits for both.
|
||||
let replication = if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
Some(tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let write_result = write_upload_needle(&state, vid, n, fsync).await;
|
||||
|
||||
// Replicate to remote volume servers if this volume has replicas.
|
||||
// Matches Go's GetWritableRemoteReplications: skip if copy_count == 1.
|
||||
// No leader-known gate: when the lookup cannot run, the write must fail
|
||||
// rather than be acknowledged on fewer copies than required. The local
|
||||
// write result takes precedence, matching Go's error ordering.
|
||||
let replication_result = match replication {
|
||||
Some(task) => Some(
|
||||
task.await
|
||||
// rather than be acknowledged on fewer copies than required.
|
||||
if !is_replicate && write_result.is_ok() {
|
||||
let needs_replication = {
|
||||
let store = state.store.read().unwrap();
|
||||
store
|
||||
.find_volume(vid)
|
||||
.is_some_and(|(_, v)| v.super_block().replica_placement.get_copy_count() > 1)
|
||||
};
|
||||
if needs_replication {
|
||||
let state_clone = state.clone();
|
||||
let path_clone = path.clone();
|
||||
let query_clone = query.clone();
|
||||
let headers_clone = headers.clone();
|
||||
let body_clone = body.clone();
|
||||
let replication = tokio::spawn(async move {
|
||||
do_replicated_request(
|
||||
&state_clone,
|
||||
vid.0,
|
||||
Method::POST,
|
||||
&path_clone,
|
||||
&query_clone,
|
||||
&headers_clone,
|
||||
Some(body_clone),
|
||||
)
|
||||
.await
|
||||
});
|
||||
let replication_result = replication
|
||||
.await
|
||||
.map_err(|e| format!("replication task failed: {}", e))
|
||||
.flatten(),
|
||||
),
|
||||
None => None,
|
||||
};
|
||||
if write_result.is_ok()
|
||||
&& let Some(Err(e)) = replication_result
|
||||
{
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
.flatten();
|
||||
if let Err(e) = replication_result {
|
||||
tracing::error!("replicated write failed: {}", e);
|
||||
return json_error_with_query(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("replication failed: {}", e),
|
||||
Some(&query),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let resp = match write_result {
|
||||
|
||||
@@ -466,7 +466,6 @@ func (env *TestEnvironment) startTrinoContainer(t *testing.T, configDir string)
|
||||
|
||||
cmd := exec.Command("docker", "run", "-d",
|
||||
"--name", containerName,
|
||||
"--memory", "4g",
|
||||
"--add-host", "host.docker.internal:host-gateway",
|
||||
"-v", fmt.Sprintf("%s:/etc/trino/catalog", configDir),
|
||||
"-v", fmt.Sprintf("%s:/test", env.dataDir),
|
||||
@@ -485,7 +484,7 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
|
||||
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastOutput []byte
|
||||
containerDead := false
|
||||
retryCount := 0
|
||||
for time.Now().Before(deadline) {
|
||||
// Try system catalog query as a readiness check
|
||||
cmd := exec.Command("docker", "exec", containerName,
|
||||
@@ -496,51 +495,26 @@ func waitForTrino(t *testing.T, containerName string, timeout time.Duration) {
|
||||
return
|
||||
} else {
|
||||
lastOutput = output
|
||||
if trinoContainerExited(output) {
|
||||
containerDead = true
|
||||
outputStr := string(output)
|
||||
if strings.Contains(outputStr, "No such container") ||
|
||||
strings.Contains(outputStr, "is not running") {
|
||||
break
|
||||
}
|
||||
retryCount++
|
||||
}
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
|
||||
// If we can't connect to system catalog, try to at least connect to Trino server
|
||||
if output, err := exec.Command("docker", "exec", containerName, "trino", "--version").CombinedOutput(); err == nil {
|
||||
cmd := exec.Command("docker", "exec", containerName, "trino", "--version")
|
||||
if err := cmd.Run(); err == nil {
|
||||
// Trino process is running, even if catalog isn't ready yet
|
||||
// Give it a bit more time
|
||||
time.Sleep(5 * time.Second)
|
||||
return
|
||||
} else {
|
||||
lastOutput = output
|
||||
containerDead = containerDead || trinoContainerExited(output)
|
||||
}
|
||||
|
||||
if containerDead {
|
||||
t.Fatalf("Trino container %s exited before becoming ready\nLast output:\n%s\n%s", containerName, string(lastOutput), trinoContainerDiagnostics(containerName))
|
||||
}
|
||||
t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s\n%s", string(lastOutput), trinoContainerDiagnostics(containerName))
|
||||
}
|
||||
|
||||
func trinoContainerExited(output []byte) bool {
|
||||
s := string(output)
|
||||
return strings.Contains(s, "No such container") || strings.Contains(s, "is not running")
|
||||
}
|
||||
|
||||
func trinoContainerDiagnostics(containerName string) string {
|
||||
var b strings.Builder
|
||||
if out, err := exec.Command("docker", "inspect", "-f",
|
||||
"status={{.State.Status}} exitCode={{.State.ExitCode}} oomKilled={{.State.OOMKilled}} error={{.State.Error}}",
|
||||
containerName).CombinedOutput(); err == nil {
|
||||
fmt.Fprintf(&b, "docker inspect: %s\n", strings.TrimSpace(string(out)))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "docker inspect failed: %v (%s)\n", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
if out, err := exec.Command("docker", "logs", "--tail", "80", containerName).CombinedOutput(); err == nil {
|
||||
fmt.Fprintf(&b, "docker logs:\n%s", string(out))
|
||||
} else {
|
||||
fmt.Fprintf(&b, "docker logs failed: %v (%s)", err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return b.String()
|
||||
t.Fatalf("Timed out waiting for Trino to be ready\nLast output:\n%s", string(lastOutput))
|
||||
}
|
||||
|
||||
func runTrinoSQL(t *testing.T, containerName, sql string) string {
|
||||
|
||||
@@ -3,12 +3,6 @@
|
||||
## Goal
|
||||
Create a Go integration test suite under `test/volume_server` that validates **drop-in behavior parity** for the Volume Server HTTP and gRPC APIs, so a Rust rewrite can be verified against the current Go behavior.
|
||||
|
||||
## Current Program Focus (2026-02-16)
|
||||
- Primary execution focus has shifted to implementing native Rust volume-server parity.
|
||||
- This integration suite is now the parity gate for Rust implementation work.
|
||||
- New tests should be added only when native Rust implementation reveals uncovered Go behavior that is not yet captured.
|
||||
- Rust implementation roadmap lives in `rust/volume_server/DEV_PLAN.md`.
|
||||
|
||||
## Hard Requirements
|
||||
- Tests live under `test/volume_server`.
|
||||
- Tests are written in Go.
|
||||
@@ -1133,202 +1127,3 @@ Update this section during implementation:
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: `499` cancellation and transport-interruption branches remain pending.
|
||||
- Commit: `d1e5f390a`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust volume-server migration bootstrap wiring.
|
||||
- APIs covered: Harness now supports Go master + selectable volume binary (`VOLUME_SERVER_IMPL=rust` or `VOLUME_SERVER_BINARY`), with Rust-mode smoke coverage for representative HTTP/gRPC integration tests.
|
||||
- Profiles covered: P1 smoke and default Go matrix unchanged.
|
||||
- Gaps introduced/remaining: native Rust handler implementations are tracked in `rust/volume_server/DEV_PLAN.md`; current Rust mode is a compatibility launcher phase.
|
||||
- Commit: `7beab85c2`, `880c2e1da`, `63d08e8a9`, `d402573ea`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Validated Rust-mode compatibility path with full integration suites.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust`.
|
||||
- Profiles covered: existing integration matrix in Rust launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint and storage/RPC implementation remains pending; current mode delegates execution to Go volume server.
|
||||
- Commit: `6ce4d7ede`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added Rust proxy-supervision launcher mode and validated all volume-server integration suites in proxy mode.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` suites executed successfully with `VOLUME_SERVER_IMPL=rust` and `VOLUME_SERVER_RUST_MODE=proxy`.
|
||||
- Profiles covered: existing integration matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC logic remains pending; current proxy mode still delegates backend handlers to Go.
|
||||
- Commit: `a7f50d23b`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded GitHub CI Rust smoke job to run both launcher modes (`exec` and `proxy`).
|
||||
- APIs covered: representative HTTP/gRPC smoke flows now validated in CI for both Rust launcher execution paths.
|
||||
- Profiles covered: CI smoke profile for P1 representative calls.
|
||||
- Gaps introduced/remaining: CI still runs Rust mode as smoke scope; full-suite Rust-mode CI remains optional due runtime cost.
|
||||
- Commit: `548b3d9a3`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added JWT UI override integration coverage with explicit `access.ui=true` profile support.
|
||||
- APIs covered: `/ui/index.html` behavior under JWT profile now verifies both default auth-gated path (`401`) and explicit UI-enable override path (`200` with rendered UI).
|
||||
- Profiles covered: P3 and P3+`access.ui=true`.
|
||||
- Gaps introduced/remaining: UI exposure behavior under JWT profiles is now covered for both secured and override branches.
|
||||
- Commit: `de974c05d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added oversized upload integration coverage using explicit `-fileSizeLimitMB` profile configuration.
|
||||
- APIs covered: HTTP write path now verifies payloads larger than configured file-size limit are rejected (`400`) with limit-related error context.
|
||||
- Profiles covered: P1-derived profile with `FileSizeLimitMB=1`.
|
||||
- Gaps introduced/remaining: file-size limit rejection branch is now covered; replicate-write failure and cancellation (`499`) branches remain pending.
|
||||
- Commit: `4d61cbdee`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added gRPC EC-only metadata-read unsupported-path coverage.
|
||||
- APIs covered: `ReadNeedleMeta` now verifies the explicit unsupported branch when a volume is unmounted and only EC shards are mounted.
|
||||
- Profiles covered: P1 with EC shard generation/mount lifecycle setup.
|
||||
- Gaps introduced/remaining: low-level read metadata fault-injection and transport interruption branches remain pending.
|
||||
- Commit: `37bf9b5eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added deterministic replicated-write failure coverage for unmet replication requirements.
|
||||
- APIs covered: HTTP write path now verifies replication error handling (`500`) and no-local-commit outcome (`404` on follow-up read) when volume replication is configured (`001`) but replica set cannot be satisfied.
|
||||
- Profiles covered: P1-derived single-node profile with per-volume replication override.
|
||||
- Gaps introduced/remaining: replicated-write failure branch is now covered; cancellation (`499`) and deeper transport fault-injection branches remain pending.
|
||||
- Commit: `4835d3443`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added EC-backed `BatchDelete` positive-path integration coverage.
|
||||
- APIs covered: `BatchDelete` now validates successful EC-needle deletion (`202`) with post-delete `VolumeEcShardRead` deleted-marker verification (`IsDeleted=true`).
|
||||
- Profiles covered: P1 with EC shard generate/mount lifecycle; includes test-local gRPC offset proxy for EC internal dial-path parity.
|
||||
- Gaps introduced/remaining: EC `BatchDelete` success branch is now covered; deeper distributed failure-injection permutations remain pending.
|
||||
- Commit: `1bb40b6bc`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded HTTP conditional-header matrix for `HEAD` requests with combined header scenarios.
|
||||
- APIs covered: `HEAD` now verifies `If-Modified-Since` precedence over mismatched `If-None-Match`, and `If-None-Match` precedence when IMS is stale but ETag matches.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: combined conditional-header precedence paths for `HEAD`/`GET` are now explicitly covered.
|
||||
- Commit: `ed23e290f`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added multi-volume success-stream coverage for `ReadAllNeedles`.
|
||||
- APIs covered: `ReadAllNeedles` now verifies happy-path streaming across two existing volumes in one request, with per-volume payload validation.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: multi-volume stream boundary coverage is now present for successful reads; transport interruption branches remain pending.
|
||||
- Commit: `ab95a6ef1`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded split-port unsupported HTTP method matrix with `MKCOL` coverage.
|
||||
- APIs covered: admin/public parity for `MKCOL` (`400` on admin, passthrough `200` on public) with post-call data-integrity verification.
|
||||
- Profiles covered: P2.
|
||||
- Gaps introduced/remaining: unsupported-method sampling now covers `PATCH`, `TRACE`, `PROPFIND`, `CONNECT`, and `MKCOL`; cancellation/transport fault paths remain the main unaddressed area.
|
||||
- Commit: `2ab30900d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Hardened framework port allocation to keep volume ports within `10000..55535`.
|
||||
- APIs covered: test harness now guarantees valid `admin+10000` gRPC offset ranges, eliminating EC batch-delete flake scenarios that rely on offset dialing.
|
||||
- Profiles covered: all profiles using shared framework port allocation.
|
||||
- Gaps introduced/remaining: no new API gaps; transport cancellation/fault-injection branches remain the primary uncovered area.
|
||||
- Commit: `90e82b15c`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Re-validated full Rust proxy-mode integration suite after latest HTTP/gRPC coverage additions and framework hardening.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages.
|
||||
- Profiles covered: existing matrix in Rust proxy launcher mode.
|
||||
- Gaps introduced/remaining: native Rust endpoint/storage/RPC implementation remains pending (current Rust mode still delegates to Go backend handlers).
|
||||
- Commit: `6bb9d8bac`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added tail sender stream-cancellation interruption coverage.
|
||||
- APIs covered: `VolumeTailSender` now verifies client-side context cancellation behavior (`codes.Canceled`) after stream start/heartbeat.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: tail transport interruption coverage is now partially closed; receiver-side interruption and deeper network fault injection remain pending.
|
||||
- Commit: `27a80f760`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Added receiver-side tail stream interruption coverage for unavailable source node.
|
||||
- APIs covered: `VolumeTailReceiver` now verifies source connect/dial failure branch when `SourceVolumeServer` is unreachable.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: both sender and receiver transport interruption branches are now covered at integration level; deeper injected network faults remain pending.
|
||||
- Commit: `a5864c3eb`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded query CSV matrix with explicit CSV-payload parity coverage.
|
||||
- APIs covered: `Query` now verifies current CSV-input behavior (no streamed rows / immediate EOF) even when source needle content is valid CSV text.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: CSV parsing/selection implementation still absent in current server behavior; integration parity now locks the current no-output semantics for both JSON and CSV payload shapes.
|
||||
- Commit: `a12dd5f8d`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded ping unreachable-target matrix with volume-server target coverage.
|
||||
- APIs covered: `Ping` now explicitly verifies unreachable-target error wrapping for `target_type=volumeServer`, alongside existing master/filer variants.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: ping target-type matrix is covered for success and unreachable branches across master/filer/volume-server.
|
||||
- Commit: `b9fbb85af`
|
||||
|
||||
- Date: 2026-02-15
|
||||
- Change: Expanded deleted-read HTTP matrix with `HEAD` parity coverage.
|
||||
- APIs covered: `HEAD` with `readDeleted=true` now validates current behavior (`200`, empty body, payload-size `Content-Length`) after delete.
|
||||
- Profiles covered: P1.
|
||||
- Gaps introduced/remaining: deleted-read parity now covers both `GET` and `HEAD` semantics on local-volume path.
|
||||
- Commit: `cc80ad364`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Shifted planning priority to native Rust implementation parity.
|
||||
- APIs covered: no new API additions in this plan entry; integration suite remains the validation gate.
|
||||
- Profiles covered: unchanged.
|
||||
- Gaps introduced/remaining: primary remaining gap is native Rust handler/storage/RPC implementation replacing Go backend delegation.
|
||||
- Commit: `14c863dbf`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust launcher `native` mode bootstrap, made it the default Rust launcher mode, and expanded CI Rust smoke matrix coverage to include `native`.
|
||||
- APIs covered: full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated in `VOLUME_SERVER_RUST_MODE=native`; default Rust launcher path (`VOLUME_SERVER_IMPL=rust`) smoke-validated for HTTP and gRPC control tests.
|
||||
- Profiles covered: existing HTTP/gRPC matrix in native launcher mode; default-mode smoke checks on P1 control flows.
|
||||
- Gaps introduced/remaining: native Rust API/storage/RPC handlers still pending; current `native` mode remains a delegation bootstrap while parity replacement proceeds.
|
||||
- Commit: `70ddbee37`, `61befd10f`, `2e65966c0`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented first native Rust HTTP control handlers in `native` mode for `/status` and `/healthz` (admin listener path), with proxy fallback retained for remaining endpoints.
|
||||
- APIs covered: `/status` and `/healthz` now served by Rust-native code path in `VOLUME_SERVER_RUST_MODE=native`; full HTTP and gRPC integration suites re-validated.
|
||||
- Profiles covered: full existing HTTP/gRPC matrix under native mode.
|
||||
- Gaps introduced/remaining: gRPC control/data APIs and most HTTP data/static/auth paths remain delegated and require incremental native replacement.
|
||||
- Commit: `7e6e0261a`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented native Rust HTTP `OPTIONS` handling for admin/public listeners in `VOLUME_SERVER_RUST_MODE=native`.
|
||||
- APIs covered: admin/public `OPTIONS` method allow-list and CORS origin handling now served from Rust-native path; full HTTP and gRPC packages re-validated.
|
||||
- Profiles covered: P2 split-port CORS/OPTIONS profiles plus full existing matrix in native mode.
|
||||
- Gaps introduced/remaining: static/UI/auth and data-path HTTP handlers, plus gRPC control/data/stream handlers, remain delegated and pending native replacement.
|
||||
- Commit: `fbff2cb39`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Implemented broader native Rust HTTP surface in `native` mode:
|
||||
- `/ui/index.html` auth-gating parity (JWT + access.ui handling)
|
||||
- static asset paths (`/favicon.ico`, `/seaweedfsstatic/*`)
|
||||
- public non-read method no-op parity (`200`) and admin unsupported-method rejection parity (`400`)
|
||||
- APIs covered: UI/static/admin/public control-surface behavior now served by Rust-native path; full `/test/volume_server/http` and `/test/volume_server/grpc` packages re-validated.
|
||||
- Profiles covered: P1/P2/P3 variants touched by UI/static/CORS/method behavior, plus full matrix in native mode.
|
||||
- Gaps introduced/remaining: HTTP data-path handlers and all gRPC handlers remain delegated and still require native replacement.
|
||||
- Commit: `23e4497b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Improved native Rust control-path parity for health/status and request-target parsing.
|
||||
- APIs covered: native `/healthz` now mirrors backend service health transitions (`200`/`503`) and native route matching now handles absolute-form HTTP request targets before path dispatch.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode (`VOLUME_SERVER_IMPL=rust`, `VOLUME_SERVER_RUST_MODE=native`).
|
||||
- Gaps introduced/remaining: core HTTP data handlers and all gRPC RPC handlers are still delegated and remain the primary native implementation gap.
|
||||
- Commit: `d6ff6ed6d`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native malformed fid-route validation ahead of delegated data handlers.
|
||||
- APIs covered: GET/HEAD/POST/PUT fid-shaped paths now return native `400` for invalid vid/fid tokens (including invalid-read and invalid-write path variants) while valid routes continue through delegated storage handlers.
|
||||
- Profiles covered: full existing HTTP/gRPC integration matrix in native mode, plus targeted invalid-path parity runs.
|
||||
- Gaps introduced/remaining: data-path success branches and all gRPC handler bodies remain delegated; native replacement work continues on those core execution paths.
|
||||
- Commit: `1ce0174b2`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Added Rust-native write error prevalidation for delegated fid routes.
|
||||
- APIs covered: native path now rejects malformed multipart boundary requests, `Content-MD5` writes, and over-limit `Content-Length` writes (using `-fileSizeLimitMB`) with `400` parity behavior before delegation.
|
||||
- Profiles covered: P1 default and P1 with explicit file-size limit profile, plus full existing HTTP/gRPC native-mode matrix revalidation.
|
||||
- Gaps introduced/remaining: positive write/read/delete data handlers and all gRPC handler bodies are still delegated; native implementation continues incrementally.
|
||||
- Commit: `94cefd6f4`
|
||||
|
||||
- Date: 2026-02-16
|
||||
- Change: Broadened Rust-native malformed-fid handling to slash-form read/write URL shapes with reserved-route exclusions.
|
||||
- APIs covered: `/{vid}/{fid}` and `/{vid}/{fid}/{filename}` malformed cases now hit native `400` validation path, while `/status`, `/healthz`, `/ui/index.html`, `/stats/*`, and static asset routes remain excluded from fid parsing.
|
||||
- Profiles covered: full existing HTTP/gRPC native-mode matrix, plus targeted admin/read-path malformed-slash validation run.
|
||||
- Gaps introduced/remaining: successful data-path read/write/delete handlers and all gRPC method bodies remain delegated and are still the primary native implementation gap.
|
||||
- Commit: `e66983e9b`
|
||||
@@ -1,10 +1,7 @@
|
||||
.PHONY: test-volume-server test-volume-server-short test-volume-server-rust-smoke
|
||||
.PHONY: test-volume-server test-volume-server-short
|
||||
|
||||
test-volume-server:
|
||||
go test ./test/volume_server/... -v
|
||||
|
||||
test-volume-server-short:
|
||||
go test ./test/volume_server/... -short -v
|
||||
|
||||
test-volume-server-rust-smoke:
|
||||
VOLUME_SERVER_IMPL=rust go test ./test/volume_server/http ./test/volume_server/grpc -run 'TestAdminStatusAndHealthz|TestUploadReadRangeHeadDeleteRoundTrip|TestStateAndStatusRPCs|TestVolumeSyncStatusAndReadVolumeFileStatus' -v
|
||||
@@ -15,9 +15,6 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
## Optional environment variables
|
||||
|
||||
- `WEED_BINARY`: explicit path to the `weed` executable (disables auto-build).
|
||||
- `VOLUME_SERVER_IMPL`: select volume-server implementation (`go` default, `rust` for a Rust volume server).
|
||||
- `RUST_VOLUME_BINARY`: explicit path to the Rust volume-server executable (overrides crate auto-detect).
|
||||
- `VOLUME_SERVER_RUST_MODE`: `weed-volume-rs` launcher mode (`native` default, `exec` for direct Go delegation, `proxy` for Rust front proxy + Go backend).
|
||||
- `VOLUME_SERVER_IT_KEEP_LOGS=1`: keep temporary test directories and process logs.
|
||||
|
||||
## Current scope (Phase 0)
|
||||
@@ -27,4 +24,4 @@ If a `weed` binary is not found, the harness will build one automatically.
|
||||
- Initial HTTP admin endpoint checks
|
||||
- Initial gRPC state/status checks
|
||||
|
||||
More API coverage is tracked in `test/volume_server/DEV_PLAN.md`.
|
||||
More API coverage is tracked in `/Users/chris/dev/seaweedfs2/test/volume_server/DEV_PLAN.md`.
|
||||
@@ -287,9 +287,6 @@ func (c *Cluster) startVolume(dataDirs []string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
c.volumeCmd = exec.Command(c.weedBinary, args...)
|
||||
c.volumeCmd.Dir = c.baseDir
|
||||
|
||||
@@ -265,9 +265,6 @@ func (c *MixedVolumeCluster) startGoVolume(index int, dataDir string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
|
||||
@@ -267,9 +267,6 @@ func (c *MultiVolumeCluster) startVolume(index int, dataDirs []string) error {
|
||||
if c.profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "-inflightDownloadDataTimeout="+c.profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if c.profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "-fileSizeLimitMB="+strconv.Itoa(c.profile.FileSizeLimitMB))
|
||||
}
|
||||
|
||||
cmd := exec.Command(c.weedBinary, args...)
|
||||
cmd.Dir = c.baseDir
|
||||
|
||||
@@ -214,9 +214,6 @@ func rustVolumeArgs(
|
||||
if profile.InflightDownloadTimeout > 0 {
|
||||
args = append(args, "--inflightDownloadDataTimeout", profile.InflightDownloadTimeout.String())
|
||||
}
|
||||
if profile.FileSizeLimitMB > 0 {
|
||||
args = append(args, "--fileSizeLimitMB", strconv.Itoa(profile.FileSizeLimitMB))
|
||||
}
|
||||
return args
|
||||
}
|
||||
|
||||
@@ -255,21 +252,12 @@ func FindOrBuildRustBinary() (string, error) {
|
||||
rustBinaryOnce.Do(func() {
|
||||
// Derive the Rust volume crate directory from this source file's location.
|
||||
rustCrateDir := ""
|
||||
releaseBin := ""
|
||||
if _, file, _, ok := runtime.Caller(0); ok {
|
||||
repoRoot := filepath.Clean(filepath.Join(filepath.Dir(file), "..", "..", ".."))
|
||||
for _, candidate := range []struct {
|
||||
dir string
|
||||
bin string
|
||||
}{
|
||||
{"seaweed-volume", "weed-volume"},
|
||||
{"weed-volume", "weed-volume"},
|
||||
{filepath.Join("rust", "volume_server"), "weed-volume-rs"},
|
||||
} {
|
||||
dir := filepath.Join(repoRoot, candidate.dir)
|
||||
for _, candidate := range []string{"seaweed-volume", "weed-volume"} {
|
||||
dir := filepath.Join(repoRoot, candidate)
|
||||
if isDir(dir) && isFile(filepath.Join(dir, "Cargo.toml")) {
|
||||
rustCrateDir = dir
|
||||
releaseBin = filepath.Join(dir, "target", "release", candidate.bin)
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -279,6 +267,8 @@ func FindOrBuildRustBinary() (string, error) {
|
||||
return
|
||||
}
|
||||
|
||||
releaseBin := filepath.Join(rustCrateDir, "target", "release", "weed-volume")
|
||||
|
||||
// Always rebuild once per test process so the harness uses current source
|
||||
// and features. The crate defaults to 5bytes, so a test binary built
|
||||
// without 5BytesOffset has to turn it off or the Rust server refuses the
|
||||
|
||||
@@ -16,39 +16,31 @@ import (
|
||||
|
||||
func AllocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, "", "")
|
||||
}
|
||||
|
||||
func AllocateVolumeWithReplication(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection, replication string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, replication, "")
|
||||
allocateVolume(t, client, volumeID, collection, "")
|
||||
}
|
||||
|
||||
// AllocateVolumeWithTTL allocates a volume whose needles expire after ttl
|
||||
// (e.g. "10m").
|
||||
func AllocateVolumeWithTTL(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, ttl string) {
|
||||
t.Helper()
|
||||
allocateVolume(t, client, volumeID, collection, "", ttl)
|
||||
allocateVolume(t, client, volumeID, collection, ttl)
|
||||
}
|
||||
|
||||
func allocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, replication, ttl string) {
|
||||
func allocateVolume(t testing.TB, client volume_server_pb.VolumeServerClient, volumeID uint32, collection string, ttl string) {
|
||||
t.Helper()
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if replication == "" {
|
||||
replication = "000"
|
||||
}
|
||||
|
||||
_, err := client.AllocateVolume(ctx, &volume_server_pb.AllocateVolumeRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: collection,
|
||||
Replication: replication,
|
||||
Replication: "000",
|
||||
Ttl: ttl,
|
||||
Version: uint32(needle.GetCurrentVersion()),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("allocate volume %d (replication=%s): %v", volumeID, replication, err)
|
||||
t.Fatalf("allocate volume %d: %v", volumeID, err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -456,19 +456,6 @@ func TestPingUnknownAndUnreachableTargetPaths(t *testing.T) {
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping filer expected InvalidArgument, got %s: %v", got, err)
|
||||
}
|
||||
|
||||
// Volume-server targets are not tracked peers either, so they are
|
||||
// rejected at admission the same way filer targets are.
|
||||
_, err = grpcClient.Ping(ctx, &volume_server_pb.PingRequest{
|
||||
TargetType: cluster.VolumeServerType,
|
||||
Target: "127.0.0.1:1.1",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("Ping volume server target should be rejected by admission")
|
||||
}
|
||||
if got := status.Code(err); got != codes.InvalidArgument {
|
||||
t.Fatalf("Ping volume server expected InvalidArgument, got %s: %v", got, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPingMasterTargetSuccess(t *testing.T) {
|
||||
|
||||
@@ -3,10 +3,7 @@ package volume_server_grpc_test
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -268,141 +265,3 @@ func TestBatchDeleteRejectsChunkManifestNeedles(t *testing.T) {
|
||||
t.Fatalf("chunk manifest should not be deleted by BatchDelete reject path, got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchDeleteEcNeedleSuccess(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
cluster := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, client := framework.DialVolumeServer(t, cluster.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
if stopProxy := maybeStartGrpcOffsetProxy(t, cluster.VolumeAdminAddress(), cluster.VolumeGRPCAddress()); stopProxy != nil {
|
||||
t.Cleanup(stopProxy)
|
||||
}
|
||||
|
||||
const volumeID = uint32(34)
|
||||
const needleID = uint64(930001)
|
||||
const cookie = uint32(0x6677BBCC)
|
||||
framework.AllocateVolume(t, client, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, cluster.VolumeAdminURL(), fid, []byte("batch-delete-ec-success"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := client.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = client.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
deleteResp, err := client.BatchDelete(ctx, &volume_server_pb.BatchDeleteRequest{FileIds: []string{fid}})
|
||||
if err != nil {
|
||||
t.Fatalf("BatchDelete EC needle should return response, got grpc error: %v", err)
|
||||
}
|
||||
if len(deleteResp.GetResults()) != 1 {
|
||||
t.Fatalf("BatchDelete EC needle expected one result, got %d", len(deleteResp.GetResults()))
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetStatus() != http.StatusAccepted {
|
||||
t.Fatalf("BatchDelete EC needle expected status 202, got %d error=%q", deleteResp.GetResults()[0].GetStatus(), deleteResp.GetResults()[0].GetError())
|
||||
}
|
||||
if deleteResp.GetResults()[0].GetSize() == 0 {
|
||||
t.Fatalf("BatchDelete EC needle expected non-zero deleted size")
|
||||
}
|
||||
|
||||
deletedStream, err := client.VolumeEcShardRead(ctx, &volume_server_pb.VolumeEcShardReadRequest{
|
||||
VolumeId: volumeID,
|
||||
ShardId: 0,
|
||||
FileKey: needleID,
|
||||
Offset: 0,
|
||||
Size: 1,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check start failed: %v", err)
|
||||
}
|
||||
deletedMsg, err := deletedStream.Recv()
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardRead deleted-check recv failed: %v", err)
|
||||
}
|
||||
if !deletedMsg.GetIsDeleted() {
|
||||
t.Fatalf("VolumeEcShardRead expected IsDeleted=true after EC batch delete")
|
||||
}
|
||||
}
|
||||
|
||||
func maybeStartGrpcOffsetProxy(t testing.TB, adminAddr, actualGrpcAddr string) func() {
|
||||
t.Helper()
|
||||
|
||||
host, portText, err := net.SplitHostPort(adminAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("split admin address %q: %v", adminAddr, err)
|
||||
}
|
||||
port, err := strconv.Atoi(portText)
|
||||
if err != nil {
|
||||
t.Fatalf("parse admin port %q: %v", portText, err)
|
||||
}
|
||||
|
||||
expectedGrpcAddr := net.JoinHostPort(host, strconv.Itoa(port+10000))
|
||||
if expectedGrpcAddr == actualGrpcAddr {
|
||||
return nil
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", expectedGrpcAddr)
|
||||
if err != nil {
|
||||
t.Fatalf("listen grpc-offset proxy %s -> %s: %v", expectedGrpcAddr, actualGrpcAddr, err)
|
||||
}
|
||||
|
||||
stopped := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
conn, acceptErr := listener.Accept()
|
||||
if acceptErr != nil {
|
||||
select {
|
||||
case <-stopped:
|
||||
return
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
go proxyBidirectional(conn, actualGrpcAddr)
|
||||
}
|
||||
}()
|
||||
|
||||
return func() {
|
||||
close(stopped)
|
||||
_ = listener.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func proxyBidirectional(inbound net.Conn, targetAddr string) {
|
||||
outbound, err := net.Dial("tcp", targetAddr)
|
||||
if err != nil {
|
||||
_ = inbound.Close()
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
_, _ = io.Copy(outbound, inbound)
|
||||
_ = outbound.Close()
|
||||
}()
|
||||
_, _ = io.Copy(inbound, outbound)
|
||||
_ = inbound.Close()
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package volume_server_grpc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -145,66 +144,3 @@ func TestReadNeedleBlobAndMetaInvalidOffsets(t *testing.T) {
|
||||
t.Fatalf("ReadNeedleMeta should fail for invalid offset")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadNeedleMetaUnsupportedForEcMountedVolume(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(93)
|
||||
const needleID = uint64(880002)
|
||||
const cookie = uint32(0x11AA22BB)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, []byte("ec-meta-unsupported"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeEcShardsGenerate(ctx, &volume_server_pb.VolumeEcShardsGenerateRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsGenerate failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeEcShardsMount(ctx, &volume_server_pb.VolumeEcShardsMountRequest{
|
||||
VolumeId: volumeID,
|
||||
Collection: "",
|
||||
ShardIds: []uint32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeEcShardsMount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.VolumeUnmount(ctx, &volume_server_pb.VolumeUnmountRequest{
|
||||
VolumeId: volumeID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("VolumeUnmount failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = grpcClient.ReadNeedleMeta(ctx, &volume_server_pb.ReadNeedleMetaRequest{
|
||||
VolumeId: volumeID,
|
||||
NeedleId: needleID,
|
||||
Offset: 0,
|
||||
Size: 128,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("ReadNeedleMeta should fail when only EC shards are mounted")
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(err.Error()), "ec shards is not supported") {
|
||||
t.Fatalf("ReadNeedleMeta EC-only error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -426,67 +426,3 @@ func findLastNeedleOffsetAndSize(t testing.TB, idxBytes []byte, needleID uint64)
|
||||
}
|
||||
return offset, size
|
||||
}
|
||||
func TestReadAllNeedlesStreamsAcrossMultipleVolumes(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeIDA = uint32(86)
|
||||
const volumeIDB = uint32(87)
|
||||
const needleIDA = uint64(445561)
|
||||
const needleIDB = uint64(445562)
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDA, "")
|
||||
framework.AllocateVolume(t, grpcClient, volumeIDB, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fidA := framework.NewFileID(volumeIDA, needleIDA, 0xAA11CC22)
|
||||
fidB := framework.NewFileID(volumeIDB, needleIDB, 0xBB22DD33)
|
||||
payloadA := "read-all-multi-volume-a"
|
||||
payloadB := "read-all-multi-volume-b"
|
||||
|
||||
uploadA := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidA, []byte(payloadA))
|
||||
_ = framework.ReadAllAndClose(t, uploadA)
|
||||
if uploadA.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload A expected 201, got %d", uploadA.StatusCode)
|
||||
}
|
||||
uploadB := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fidB, []byte(payloadB))
|
||||
_ = framework.ReadAllAndClose(t, uploadB)
|
||||
if uploadB.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload B expected 201, got %d", uploadB.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
stream, err := grpcClient.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
|
||||
VolumeIds: []uint32{volumeIDA, volumeIDB},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadAllNeedles start failed: %v", err)
|
||||
}
|
||||
|
||||
seen := map[uint64]string{}
|
||||
for {
|
||||
msg, recvErr := stream.Recv()
|
||||
if recvErr == io.EOF {
|
||||
break
|
||||
}
|
||||
if recvErr != nil {
|
||||
t.Fatalf("ReadAllNeedles recv failed: %v", recvErr)
|
||||
}
|
||||
if msg.GetNeedleId() == needleIDA || msg.GetNeedleId() == needleIDB {
|
||||
seen[msg.GetNeedleId()] = string(msg.GetNeedleBlob())
|
||||
}
|
||||
}
|
||||
|
||||
if got := seen[needleIDA]; got != payloadA {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume A needle: got %q want %q", got, payloadA)
|
||||
}
|
||||
if got := seen[needleIDB]; got != payloadB {
|
||||
t.Fatalf("ReadAllNeedles missing/mismatched payload for volume B needle: got %q want %q", got, payloadB)
|
||||
}
|
||||
}
|
||||
@@ -280,54 +280,6 @@ func TestQueryJsonSuccessAndCsvNoOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryCsvInputWithCsvPayloadStillReturnsNoRows(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(67)
|
||||
const needleID = uint64(777004)
|
||||
const cookie = uint32(0xDCDCADAD)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
csvPayload := []byte("name,score\nalice,12\nbob,3\n")
|
||||
httpClient := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, needleID, cookie)
|
||||
uploadResp := framework.UploadBytes(t, httpClient, clusterHarness.VolumeAdminURL(), fid, csvPayload)
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != 201 {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
csvStream, err := grpcClient.Query(ctx, &volume_server_pb.QueryRequest{
|
||||
FromFileIds: []string{fid},
|
||||
Selections: []string{"score"},
|
||||
Filter: &volume_server_pb.QueryRequest_Filter{
|
||||
Field: "score",
|
||||
Operand: ">",
|
||||
Value: "10",
|
||||
},
|
||||
InputSerialization: &volume_server_pb.QueryRequest_InputSerialization{
|
||||
CsvInput: &volume_server_pb.QueryRequest_InputSerialization_CSVInput{},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Query csv start failed: %v", err)
|
||||
}
|
||||
|
||||
_, err = csvStream.Recv()
|
||||
if err != io.EOF {
|
||||
t.Fatalf("Query csv with csv payload expected EOF with no rows, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryJsonNoMatchReturnsEmptyStripe(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
|
||||
@@ -12,8 +12,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/matrix"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
func TestVolumeTailSenderMissingVolume(t *testing.T) {
|
||||
@@ -93,36 +91,6 @@ func TestVolumeTailReceiverMissingVolume(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverSourceUnavailable(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(89)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
_, err := grpcClient.VolumeTailReceiver(ctx, &volume_server_pb.VolumeTailReceiverRequest{
|
||||
VolumeId: volumeID,
|
||||
SourceVolumeServer: "127.0.0.1:19999.29999",
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 1,
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailReceiver should fail when source volume server is unavailable")
|
||||
}
|
||||
lowered := strings.ToLower(err.Error())
|
||||
if !strings.Contains(lowered, "dial") && !strings.Contains(lowered, "unavailable") && !strings.Contains(lowered, "connection refused") {
|
||||
t.Fatalf("VolumeTailReceiver source-unavailable error mismatch: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailReceiverReplicatesSourceUpdates(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
@@ -236,46 +204,3 @@ func TestVolumeTailSenderLargeNeedleChunking(t *testing.T) {
|
||||
t.Fatalf("VolumeTailSender expected a final data chunk marked IsLastChunk=true")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVolumeTailSenderStreamCancellation(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(74)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
stream, err := grpcClient.VolumeTailSender(ctx, &volume_server_pb.VolumeTailSenderRequest{
|
||||
VolumeId: volumeID,
|
||||
SinceNs: 0,
|
||||
IdleTimeoutSeconds: 30,
|
||||
})
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender start failed: %v", err)
|
||||
}
|
||||
|
||||
firstMsg, err := stream.Recv()
|
||||
if err != nil {
|
||||
cancel()
|
||||
t.Fatalf("VolumeTailSender first recv failed: %v", err)
|
||||
}
|
||||
if !firstMsg.GetIsLastChunk() {
|
||||
cancel()
|
||||
t.Fatalf("expected heartbeat first message before cancellation")
|
||||
}
|
||||
|
||||
cancel()
|
||||
_, err = stream.Recv()
|
||||
if err == nil {
|
||||
t.Fatalf("VolumeTailSender recv after cancellation expected error")
|
||||
}
|
||||
if status.Code(err) != codes.Canceled {
|
||||
t.Fatalf("VolumeTailSender cancellation expected grpc code Canceled, got %v (%v)", status.Code(err), err)
|
||||
}
|
||||
}
|
||||
@@ -174,26 +174,6 @@ func TestUiIndexNotExposedWhenJwtSigningEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestUiIndexExposedWhenJwtSigningEnabledAndAccessUITrue(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P3()
|
||||
profile.EnableUIAccess = true
|
||||
cluster := framework.StartSingleVolumeCluster(t, profile)
|
||||
client := framework.NewHTTPClient()
|
||||
|
||||
resp := framework.DoRequest(t, client, mustNewRequest(t, http.MethodGet, cluster.VolumeAdminURL()+"/ui/index.html"))
|
||||
body := framework.ReadAllAndClose(t, resp)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("expected /ui/index.html to be exposed when access.ui=true under JWT profile, got %d body=%s", resp.StatusCode, string(body))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(body)), "volume") {
|
||||
t.Fatalf("ui page does not look like volume status page")
|
||||
}
|
||||
}
|
||||
|
||||
func mustNewRequest(t testing.TB, method, url string) *http.Request {
|
||||
t.Helper()
|
||||
req, err := http.NewRequest(method, url, nil)
|
||||
|
||||
@@ -241,90 +241,3 @@ func TestPublicPortHeadReadParity(t *testing.T) {
|
||||
t.Fatalf("public HEAD body should be empty, got %d bytes", len(headBody))
|
||||
}
|
||||
}
|
||||
func TestUnsupportedMethodConnectParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(85)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 124001, 0x03030303)
|
||||
client := framework.NewHTTPClient()
|
||||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("connect-method-check"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
adminReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
adminResp := framework.DoRequest(t, client, adminReq)
|
||||
_ = framework.ReadAllAndClose(t, adminResp)
|
||||
if adminResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("admin CONNECT expected 400, got %d", adminResp.StatusCode)
|
||||
}
|
||||
|
||||
publicReq := mustNewRequest(t, "CONNECT", clusterHarness.VolumePublicURL()+"/"+fid)
|
||||
publicResp := framework.DoRequest(t, client, publicReq)
|
||||
_ = framework.ReadAllAndClose(t, publicResp)
|
||||
if publicResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("public CONNECT expected passthrough 200, got %d", publicResp.StatusCode)
|
||||
}
|
||||
|
||||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
verifyBody := framework.ReadAllAndClose(t, verifyResp)
|
||||
if verifyResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
|
||||
}
|
||||
if string(verifyBody) != "connect-method-check" {
|
||||
t.Fatalf("CONNECT should not mutate data, got %q", string(verifyBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnsupportedMethodMkcolParity(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartVolumeCluster(t, matrix.P2())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(88)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
fid := framework.NewFileID(volumeID, 124004, 0x06060606)
|
||||
client := framework.NewHTTPClient()
|
||||
uploadResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, []byte("mkcol-method-check"))
|
||||
_ = framework.ReadAllAndClose(t, uploadResp)
|
||||
if uploadResp.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("upload expected 201, got %d", uploadResp.StatusCode)
|
||||
}
|
||||
|
||||
adminReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
adminResp := framework.DoRequest(t, client, adminReq)
|
||||
_ = framework.ReadAllAndClose(t, adminResp)
|
||||
if adminResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("admin MKCOL expected 400, got %d", adminResp.StatusCode)
|
||||
}
|
||||
|
||||
publicReq := mustNewRequest(t, "MKCOL", clusterHarness.VolumePublicURL()+"/"+fid)
|
||||
publicResp := framework.DoRequest(t, client, publicReq)
|
||||
_ = framework.ReadAllAndClose(t, publicResp)
|
||||
if publicResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("public MKCOL expected passthrough 200, got %d", publicResp.StatusCode)
|
||||
}
|
||||
|
||||
verifyResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
verifyBody := framework.ReadAllAndClose(t, verifyResp)
|
||||
if verifyResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("verify GET expected 200, got %d", verifyResp.StatusCode)
|
||||
}
|
||||
if string(verifyBody) != "mkcol-method-check" {
|
||||
t.Fatalf("MKCOL should not mutate data, got %q", string(verifyBody))
|
||||
}
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package volume_server_http_test
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/test/volume_server/framework"
|
||||
@@ -52,17 +51,4 @@ func TestReadDeletedQueryReturnsDeletedNeedleData(t *testing.T) {
|
||||
if string(readDeletedBody) != string(payload) {
|
||||
t.Fatalf("readDeleted body mismatch: got %q want %q", string(readDeletedBody), string(payload))
|
||||
}
|
||||
|
||||
headReadDeletedReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid+"?readDeleted=true")
|
||||
headReadDeletedResp := framework.DoRequest(t, client, headReadDeletedReq)
|
||||
headReadDeletedBody := framework.ReadAllAndClose(t, headReadDeletedResp)
|
||||
if headReadDeletedResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD with readDeleted=true expected 200, got %d", headReadDeletedResp.StatusCode)
|
||||
}
|
||||
if len(headReadDeletedBody) != 0 {
|
||||
t.Fatalf("HEAD with readDeleted=true expected empty body, got %d bytes", len(headReadDeletedBody))
|
||||
}
|
||||
if got := headReadDeletedResp.Header.Get("Content-Length"); got != strconv.Itoa(len(payload)) {
|
||||
t.Fatalf("HEAD with readDeleted=true content-length mismatch: got %q want %d", got, len(payload))
|
||||
}
|
||||
}
|
||||
@@ -164,21 +164,17 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if lastModified == "" {
|
||||
t.Fatalf("baseline read expected Last-Modified header")
|
||||
}
|
||||
etag := baselineResp.Header.Get("ETag")
|
||||
if etag == "" {
|
||||
t.Fatalf("baseline read expected ETag header")
|
||||
}
|
||||
|
||||
precedenceReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
precedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
precedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
|
||||
precedenceResp := framework.DoRequest(t, client, precedenceReq)
|
||||
precedenceBody := framework.ReadAllAndClose(t, precedenceResp)
|
||||
if precedenceResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("mismatched If-None-Match overrides If-Modified-Since: expected 200, got %d", precedenceResp.StatusCode)
|
||||
if precedenceResp.StatusCode != http.StatusNotModified {
|
||||
t.Fatalf("conditional precedence expected 304, got %d", precedenceResp.StatusCode)
|
||||
}
|
||||
if string(precedenceBody) != string(payload) {
|
||||
t.Fatalf("mismatched If-None-Match body mismatch: got %q want %q", string(precedenceBody), string(payload))
|
||||
if len(precedenceBody) != 0 {
|
||||
t.Fatalf("conditional precedence expected empty body, got %d bytes", len(precedenceBody))
|
||||
}
|
||||
|
||||
invalidIMSReq := mustNewRequest(t, http.MethodGet, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
@@ -192,28 +188,4 @@ func TestConditionalHeaderPrecedenceAndInvalidIfModifiedSince(t *testing.T) {
|
||||
if string(invalidIMSBody) != string(payload) {
|
||||
t.Fatalf("invalid If-Modified-Since fallback body mismatch: got %q want %q", string(invalidIMSBody), string(payload))
|
||||
}
|
||||
|
||||
headIMSPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headIMSPrecedenceReq.Header.Set("If-Modified-Since", lastModified)
|
||||
headIMSPrecedenceReq.Header.Set("If-None-Match", "\"definitely-different-etag\"")
|
||||
headIMSPrecedenceResp := framework.DoRequest(t, client, headIMSPrecedenceReq)
|
||||
headIMSPrecedenceBody := framework.ReadAllAndClose(t, headIMSPrecedenceResp)
|
||||
if headIMSPrecedenceResp.StatusCode != http.StatusOK {
|
||||
t.Fatalf("HEAD mismatched If-None-Match overrides If-Modified-Since: expected 200, got %d", headIMSPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headIMSPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional precedence expected empty body, got %d bytes", len(headIMSPrecedenceBody))
|
||||
}
|
||||
|
||||
headETagPrecedenceReq := mustNewRequest(t, http.MethodHead, clusterHarness.VolumeAdminURL()+"/"+fid)
|
||||
headETagPrecedenceReq.Header.Set("If-Modified-Since", "Thu, 01 Jan 1970 00:00:00 GMT")
|
||||
headETagPrecedenceReq.Header.Set("If-None-Match", etag)
|
||||
headETagPrecedenceResp := framework.DoRequest(t, client, headETagPrecedenceReq)
|
||||
headETagPrecedenceBody := framework.ReadAllAndClose(t, headETagPrecedenceResp)
|
||||
if headETagPrecedenceResp.StatusCode != http.StatusNotModified {
|
||||
t.Fatalf("HEAD conditional precedence expected 304 when If-None-Match matches, got %d", headETagPrecedenceResp.StatusCode)
|
||||
}
|
||||
if len(headETagPrecedenceBody) != 0 {
|
||||
t.Fatalf("HEAD conditional etag precedence expected empty body, got %d bytes", len(headETagPrecedenceBody))
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
package volume_server_http_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -73,64 +72,3 @@ func TestWriteMalformedMultipartAndMD5Mismatch(t *testing.T) {
|
||||
t.Fatalf("content-md5 mismatch response should mention Content-MD5, got %q", string(md5MismatchBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteRejectsPayloadOverFileSizeLimit(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
profile := matrix.P1()
|
||||
profile.FileSizeLimitMB = 1
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, profile)
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(99)
|
||||
framework.AllocateVolume(t, grpcClient, volumeID, "")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772002, 0x2A3B4C5D)
|
||||
oversizedPayload := bytes.Repeat([]byte("z"), 1024*1024+1)
|
||||
|
||||
oversizedReq := newUploadRequest(t, clusterHarness.VolumeAdminURL()+"/"+fid, oversizedPayload)
|
||||
oversizedResp := framework.DoRequest(t, client, oversizedReq)
|
||||
oversizedBody := framework.ReadAllAndClose(t, oversizedResp)
|
||||
if oversizedResp.StatusCode != http.StatusBadRequest {
|
||||
t.Fatalf("oversized write expected 400, got %d", oversizedResp.StatusCode)
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(oversizedBody)), "limited") {
|
||||
t.Fatalf("oversized write response should mention limit, got %q", string(oversizedBody))
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicatedWriteFailsWhenReplicaRequirementsNotMet(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test in short mode")
|
||||
}
|
||||
|
||||
clusterHarness := framework.StartSingleVolumeCluster(t, matrix.P1())
|
||||
conn, grpcClient := framework.DialVolumeServer(t, clusterHarness.VolumeGRPCAddress())
|
||||
defer conn.Close()
|
||||
|
||||
const volumeID = uint32(109)
|
||||
framework.AllocateVolumeWithReplication(t, grpcClient, volumeID, "", "001")
|
||||
|
||||
client := framework.NewHTTPClient()
|
||||
fid := framework.NewFileID(volumeID, 772003, 0x3A4B5C6D)
|
||||
payload := []byte("replicated-write-failure-path")
|
||||
|
||||
writeResp := framework.UploadBytes(t, client, clusterHarness.VolumeAdminURL(), fid, payload)
|
||||
writeBody := framework.ReadAllAndClose(t, writeResp)
|
||||
if writeResp.StatusCode != http.StatusInternalServerError {
|
||||
t.Fatalf("replicated write with unmet replication requirements expected 500, got %d body=%s", writeResp.StatusCode, string(writeBody))
|
||||
}
|
||||
if !strings.Contains(strings.ToLower(string(writeBody)), "replica") {
|
||||
t.Fatalf("replicated write failure response should mention replica write failure, got %q", string(writeBody))
|
||||
}
|
||||
|
||||
readResp := framework.ReadBytes(t, client, clusterHarness.VolumeAdminURL(), fid)
|
||||
_ = framework.ReadAllAndClose(t, readResp)
|
||||
if readResp.StatusCode != http.StatusNotFound {
|
||||
t.Fatalf("local read after failed replicate write expected 404 (not committed locally), got %d", readResp.StatusCode)
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,6 @@ type Profile struct {
|
||||
ConcurrentDownloadLimitMB int
|
||||
InflightUploadTimeout time.Duration
|
||||
InflightDownloadTimeout time.Duration
|
||||
FileSizeLimitMB int
|
||||
|
||||
ReplicatedLayout bool
|
||||
HasErasureCoding bool
|
||||
|
||||
+3
-25
@@ -3,7 +3,6 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
httppprof "net/http/pprof"
|
||||
"os"
|
||||
@@ -70,7 +69,6 @@ type VolumeServerOptions struct {
|
||||
preStopSeconds *int
|
||||
metricsHttpPort *int
|
||||
metricsHttpIp *string
|
||||
proxyProtocol *bool
|
||||
// pulseSeconds *int
|
||||
inflightUploadDataTimeout *time.Duration
|
||||
inflightDownloadDataTimeout *time.Duration
|
||||
@@ -221,7 +219,6 @@ func init() {
|
||||
v.id = cmdVolume.Flag.String("id", "", "volume server id. If empty, default to ip:port")
|
||||
v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
|
||||
v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to. If empty, default to same as -ip option.")
|
||||
v.proxyProtocol = cmdVolume.Flag.Bool("proxyProtocol", false, "accept PROXY protocol v1 headers on volume listeners; enable only behind a trusted proxy")
|
||||
v.mastersString = cmdVolume.Flag.String("master", "localhost:9333", "comma-separated master servers")
|
||||
v.mserverString = cmdVolume.Flag.String("mserver", "", "comma-separated master servers (deprecated, use -master instead)")
|
||||
v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
|
||||
@@ -635,23 +632,12 @@ func (m multiHttpdownServer) Stop() error {
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// acceptProxyProtocol reports whether volume listeners should unwrap PROXY
|
||||
// v1 headers; the flag pointer is nil for embedded starts (server/mini)
|
||||
// that do not register it.
|
||||
func (v VolumeServerOptions) acceptProxyProtocol() bool {
|
||||
return v.proxyProtocol != nil && *v.proxyProtocol
|
||||
}
|
||||
|
||||
func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
|
||||
grpcPort := *v.portGrpc
|
||||
grpcL, grpcLocalL, err := util.NewIpAndLocalListeners(*v.bindIp, grpcPort, 0)
|
||||
if err != nil {
|
||||
glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
|
||||
}
|
||||
var grpcListener net.Listener = grpcL
|
||||
if v.acceptProxyProtocol() {
|
||||
grpcListener = util.WrapProxyProtocolListener(grpcListener)
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.volume"))
|
||||
volume_server_pb.RegisterVolumeServerServer(grpcS, vs)
|
||||
reflection.Register(grpcS)
|
||||
@@ -659,7 +645,7 @@ func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerSe
|
||||
go grpcS.Serve(grpcLocalL)
|
||||
}
|
||||
go func() {
|
||||
if err := grpcS.Serve(grpcListener); err != nil {
|
||||
if err := grpcS.Serve(grpcL); err != nil {
|
||||
glog.Fatalf("start gRPC service failed, %s", err)
|
||||
}
|
||||
}()
|
||||
@@ -674,13 +660,9 @@ func (v VolumeServerOptions) startPublicHttpService(handler http.Handler) httpdo
|
||||
if e != nil {
|
||||
glog.Fatalf("Volume server listener error:%v", e)
|
||||
}
|
||||
var publicHttpListener net.Listener = publicListener
|
||||
if v.acceptProxyProtocol() {
|
||||
publicHttpListener = util.WrapProxyProtocolListener(publicHttpListener)
|
||||
}
|
||||
|
||||
pubHttp := httpdown.HTTP{StopTimeout: 5 * time.Minute, KillTimeout: 5 * time.Minute}
|
||||
publicHttpDown := multiHttpdownServer{pubHttp.Serve(&http.Server{Handler: handler}, publicHttpListener)}
|
||||
publicHttpDown := multiHttpdownServer{pubHttp.Serve(&http.Server{Handler: handler}, publicListener)}
|
||||
if publicLocalListener != nil {
|
||||
publicHttpDown = append(publicHttpDown, pubHttp.Serve(&http.Server{Handler: handler}, publicLocalListener))
|
||||
}
|
||||
@@ -742,11 +724,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) (http
|
||||
httpS.TLSConfig.GetCertificate = getCert
|
||||
}
|
||||
|
||||
var clusterHttpListener net.Listener = listener
|
||||
if v.acceptProxyProtocol() {
|
||||
clusterHttpListener = util.WrapProxyProtocolListener(clusterHttpListener)
|
||||
}
|
||||
clusterHttpServers := multiHttpdownServer{httpDown.Serve(httpS, clusterHttpListener)}
|
||||
clusterHttpServers := multiHttpdownServer{httpDown.Serve(httpS, listener)}
|
||||
if localListener != nil {
|
||||
localHttpS := &http.Server{Handler: handler, TLSConfig: httpS.TLSConfig}
|
||||
clusterHttpServers = append(clusterHttpServers, httpDown.Serve(localHttpS, localListener))
|
||||
|
||||
@@ -181,7 +181,7 @@ func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_
|
||||
func (store *EtcdStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath weed_util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
|
||||
}
|
||||
|
||||
|
||||
@@ -84,6 +84,32 @@ func TestDoMaybeManifestize(t *testing.T) {
|
||||
actual, _ := doMaybeManifestize(nil, mtest.inputs, 2, mockMerge)
|
||||
assertEqualChunks(t, mtest.expected, actual)
|
||||
}
|
||||
}
|
||||
|
||||
// A mid-run merge failure returns the caller's list untouched: never a
|
||||
// half-folded list, which would drop manifests the caller came in with. The
|
||||
// MaybeManifestize wrapper deletes the manifest blobs already saved.
|
||||
func TestDoMaybeManifestizePartialFailure(t *testing.T) {
|
||||
inputs := []*filer_pb.FileChunk{
|
||||
{FileId: "0", IsChunkManifest: true},
|
||||
{FileId: "1", IsChunkManifest: false},
|
||||
{FileId: "2", IsChunkManifest: false},
|
||||
{FileId: "3", IsChunkManifest: false},
|
||||
{FileId: "4", IsChunkManifest: false},
|
||||
}
|
||||
calls := 0
|
||||
failingMerge := func(saveFunc SaveDataAsChunkFunctionType, dataChunks []*filer_pb.FileChunk) (*filer_pb.FileChunk, error) {
|
||||
calls++
|
||||
if calls > 1 {
|
||||
return nil, fmt.Errorf("merge failed")
|
||||
}
|
||||
return mockMerge(saveFunc, dataChunks)
|
||||
}
|
||||
actual, err := doMaybeManifestize(nil, inputs, 2, failingMerge)
|
||||
if err == nil {
|
||||
t.Fatalf("doMaybeManifestize() expected an error")
|
||||
}
|
||||
assertEqualChunks(t, inputs, actual)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -400,39 +400,46 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
|
||||
return actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, eachEntryFunc)
|
||||
}
|
||||
|
||||
// lastFileName must stay at the last name delivered to eachEntryFunc; the
|
||||
// raw scan cursor runs ahead over non-matching names and cannot be returned.
|
||||
var notPrefixed []*Entry
|
||||
lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit, func(entry *Entry) (bool, error) {
|
||||
notPrefixed = append(notPrefixed, entry)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
count := int64(0)
|
||||
scanName, scanInclusive := startFileName, includeStartFile
|
||||
for count < limit {
|
||||
var page []*Entry
|
||||
if scanName, err = actualStore.ListDirectoryEntries(ctx, dirPath, scanName, scanInclusive, limit, func(entry *Entry) (bool, error) {
|
||||
page = append(page, entry)
|
||||
return true, nil
|
||||
}); err != nil {
|
||||
return
|
||||
}
|
||||
scanInclusive = false
|
||||
if len(page) == 0 {
|
||||
return
|
||||
}
|
||||
for _, entry := range page {
|
||||
if !strings.HasPrefix(entry.Name(), prefix) {
|
||||
continue
|
||||
for count < limit && len(notPrefixed) > 0 {
|
||||
for _, entry := range notPrefixed {
|
||||
if strings.HasPrefix(entry.Name(), prefix) {
|
||||
count++
|
||||
res, resErr := eachEntryFunc(entry)
|
||||
|
||||
if resErr != nil {
|
||||
err = fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.Name(), resErr)
|
||||
return
|
||||
}
|
||||
|
||||
if !res {
|
||||
return
|
||||
}
|
||||
if count >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
count++
|
||||
res, resErr := eachEntryFunc(entry)
|
||||
if resErr != nil {
|
||||
err = fmt.Errorf("failed to process eachEntryFunc for entry %q: %w", entry.Name(), resErr)
|
||||
}
|
||||
if count < limit && lastFileName < prefix {
|
||||
notPrefixed = notPrefixed[:0]
|
||||
lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) (bool, error) {
|
||||
notPrefixed = append(notPrefixed, entry)
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
lastFileName = entry.Name()
|
||||
if !res || count >= limit {
|
||||
return
|
||||
}
|
||||
}
|
||||
if scanName >= prefix && !strings.HasPrefix(scanName, prefix) {
|
||||
return
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
|
||||
@@ -205,41 +205,3 @@ func TestFilerStoreWrapperRollbackSucceedsWithCancelledContext(t *testing.T) {
|
||||
assert.NoError(t, wrapper.RollbackTransaction(cancelledCtx()))
|
||||
assert.NoError(t, wrapper.RollbackTransaction(expiredCtx()))
|
||||
}
|
||||
|
||||
type noPrefixListStore struct {
|
||||
*stubFilerStore
|
||||
}
|
||||
|
||||
func (s *noPrefixListStore) ListDirectoryPrefixedEntries(context.Context, util.FullPath, string, bool, int64, string, ListEachEntryFunc) (string, error) {
|
||||
return "", ErrUnsupportedListDirectoryPrefixed
|
||||
}
|
||||
|
||||
// The wrapper fallback must return the last delivered name as the cursor, not
|
||||
// the end of the raw page, and must keep paging while prefix matches can still
|
||||
// follow the raw cursor.
|
||||
func TestPrefixFilterEntriesCursorStaysAtLastDelivered(t *testing.T) {
|
||||
store := &noPrefixListStore{newStubFilerStore()}
|
||||
wrapper := NewFilerStoreWrapper(store)
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"a", "b1", "b2", "b3", "b4"} {
|
||||
require.NoError(t, store.InsertEntry(ctx, &Entry{FullPath: util.FullPath("/d/" + name)}))
|
||||
}
|
||||
|
||||
var delivered []string
|
||||
lastFileName, err := wrapper.ListDirectoryPrefixedEntries(ctx, util.FullPath("/d"), "", false, 2, "b", func(entry *Entry) (bool, error) {
|
||||
delivered = append(delivered, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"b1", "b2"}, delivered)
|
||||
assert.Equal(t, "b2", lastFileName)
|
||||
|
||||
delivered = delivered[:0]
|
||||
lastFileName, err = wrapper.ListDirectoryPrefixedEntries(ctx, util.FullPath("/d"), lastFileName, false, 10, "b", func(entry *Entry) (bool, error) {
|
||||
delivered = append(delivered, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []string{"b3", "b4"}, delivered)
|
||||
assert.Equal(t, "b4", lastFileName)
|
||||
}
|
||||
@@ -156,11 +156,7 @@ func (store *HbaseStore) ListDirectoryEntries(ctx context.Context, dirPath util.
|
||||
func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
|
||||
family := map[string][]string{store.cfMetaDir: {COLUMN_NAME}}
|
||||
expectedPrefix := []byte(dirPath.Child(prefix))
|
||||
scanStart := expectedPrefix
|
||||
if resumeKey := []byte(dirPath.Child(startFileName)); bytes.Compare(resumeKey, scanStart) > 0 {
|
||||
scanStart = resumeKey
|
||||
}
|
||||
scan, err := hrpc.NewScanRange(ctx, store.table, scanStart, nil, hrpc.Families(family))
|
||||
scan, err := hrpc.NewScanRange(ctx, store.table, expectedPrefix, nil, hrpc.Families(family))
|
||||
if err != nil {
|
||||
return lastFileName, err
|
||||
}
|
||||
@@ -192,7 +188,7 @@ func (store *HbaseStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPa
|
||||
|
||||
value := cell.Value
|
||||
|
||||
if fileName < startFileName || (fileName == startFileName && !includeStartFile) {
|
||||
if fileName == startFileName && !includeStartFile {
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
@@ -221,7 +221,7 @@ func (store *LevelDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
|
||||
|
||||
directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
|
||||
}
|
||||
|
||||
|
||||
@@ -110,28 +110,3 @@ func BenchmarkInsertEntry(b *testing.B) {
|
||||
store.InsertEntry(ctx, entry)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) {
|
||||
store := &LevelDBStore{}
|
||||
store.initialize(t.TempDir())
|
||||
defer store.Shutdown()
|
||||
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"item-a", "item-a.versions", "item-b.versions"} {
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: util.FullPath("/d/" + name)}); err != nil {
|
||||
t.Fatalf("insert %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
var names []string
|
||||
_, err := store.ListDirectoryPrefixedEntries(ctx, util.FullPath("/d"), "item", true, 100, "item-b", func(entry *filer.Entry) (bool, error) {
|
||||
names = append(names, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(names) != 1 || names[0] != "item-b.versions" {
|
||||
t.Fatalf("want [item-b.versions], got %v", names)
|
||||
}
|
||||
}
|
||||
@@ -239,7 +239,7 @@ func (store *LevelDB2Store) ListDirectoryPrefixedEntries(ctx context.Context, di
|
||||
|
||||
directoryPrefix, partitionId := genDirectoryKeyPrefix(dirPath, prefix, store.dbCount)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart, _ = genDirectoryKeyPrefix(dirPath, startFileName, store.dbCount)
|
||||
}
|
||||
|
||||
|
||||
@@ -151,31 +151,6 @@ func TestInitializeRejectsOversizedOption(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) {
|
||||
store := &LevelDB2Store{}
|
||||
store.initialize(t.TempDir(), 2)
|
||||
defer store.Shutdown()
|
||||
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"item-a", "item-a.versions", "item-b.versions"} {
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: util.FullPath("/d/" + name)}); err != nil {
|
||||
t.Fatalf("insert %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
var names []string
|
||||
_, err := store.ListDirectoryPrefixedEntries(ctx, util.FullPath("/d"), "item", true, 100, "item-b", func(entry *filer.Entry) (bool, error) {
|
||||
names = append(names, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(names) != 1 || names[0] != "item-b.versions" {
|
||||
t.Fatalf("want [item-b.versions], got %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
|
||||
@@ -370,7 +370,7 @@ func (store *LevelDB3Store) ListDirectoryPrefixedEntries(ctx context.Context, di
|
||||
|
||||
directoryPrefix := genDirectoryKeyPrefix(shortPath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart = genDirectoryKeyPrefix(shortPath, startFileName)
|
||||
}
|
||||
|
||||
|
||||
@@ -182,28 +182,3 @@ func TestEmptyRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestListDirectoryPrefixedEntries_StartBelowPrefix(t *testing.T) {
|
||||
store := &LevelDB3Store{}
|
||||
store.initialize(t.TempDir())
|
||||
defer store.Shutdown()
|
||||
|
||||
ctx := context.Background()
|
||||
for _, name := range []string{"item-a", "item-a.versions", "item-b.versions"} {
|
||||
if err := store.InsertEntry(ctx, &filer.Entry{FullPath: util.FullPath("/d/" + name)}); err != nil {
|
||||
t.Fatalf("insert %s: %v", name, err)
|
||||
}
|
||||
}
|
||||
|
||||
var names []string
|
||||
_, err := store.ListDirectoryPrefixedEntries(ctx, util.FullPath("/d"), "item", true, 100, "item-b", func(entry *filer.Entry) (bool, error) {
|
||||
names = append(names, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("list: %v", err)
|
||||
}
|
||||
if len(names) != 1 || names[0] != "item-b.versions" {
|
||||
t.Fatalf("want [item-b.versions], got %v", names)
|
||||
}
|
||||
}
|
||||
@@ -15,10 +15,6 @@ import (
|
||||
|
||||
const (
|
||||
DIR_LIST_MARKER = "\x00"
|
||||
|
||||
// a listing whose callback stops early reads at most one batch more than
|
||||
// it uses, and no single pipeline grows with the listing limit
|
||||
listFetchBatchSize = 128
|
||||
)
|
||||
|
||||
type UniversalRedis2Store struct {
|
||||
@@ -100,12 +96,7 @@ func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer
|
||||
|
||||
func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
|
||||
return decodeEntry(fullpath, store.Client.Get(ctx, store.getKey(string(fullpath))))
|
||||
}
|
||||
|
||||
func decodeEntry(fullpath util.FullPath, get *redis.StringCmd) (entry *filer.Entry, err error) {
|
||||
|
||||
data, err := get.Result()
|
||||
data, err := store.Client.Get(ctx, store.getKey(string(fullpath))).Result()
|
||||
if err == redis.Nil {
|
||||
return nil, filer_pb.ErrNotFound
|
||||
}
|
||||
@@ -211,13 +202,9 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir
|
||||
|
||||
// fetch entry meta
|
||||
var entry *filer.Entry
|
||||
var gets []*redis.StringCmd
|
||||
for i, fileName := range members {
|
||||
if i%listFetchBatchSize == 0 {
|
||||
gets = store.getEntries(ctx, dirPath, members[i:])
|
||||
}
|
||||
for _, fileName := range members {
|
||||
path := util.NewFullPath(string(dirPath), fileName)
|
||||
entry, err = decodeEntry(path, gets[i%listFetchBatchSize])
|
||||
entry, err = store.FindEntry(ctx, path)
|
||||
lastFileName = fileName
|
||||
if err != nil {
|
||||
glog.V(0).InfofCtx(ctx, "list %s : %v", path, err)
|
||||
@@ -248,23 +235,6 @@ func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, dir
|
||||
return lastFileName, err
|
||||
}
|
||||
|
||||
// getEntries pipelines the GETs for up to listFetchBatchSize of members. Each
|
||||
// command keeps its own error, so a missing value still reads as redis.Nil and
|
||||
// fails only that member.
|
||||
func (store *UniversalRedis2Store) getEntries(ctx context.Context, dirPath util.FullPath, members []string) []*redis.StringCmd {
|
||||
if len(members) > listFetchBatchSize {
|
||||
members = members[:listFetchBatchSize]
|
||||
}
|
||||
gets := make([]*redis.StringCmd, len(members))
|
||||
store.Client.Pipelined(ctx, func(pipe redis.Pipeliner) error {
|
||||
for i, fileName := range members {
|
||||
gets[i] = pipe.Get(ctx, store.getKey(string(util.NewFullPath(string(dirPath), fileName))))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return gets
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) removeOrphanedDirectoryListMember(ctx context.Context, dirPath util.FullPath, fileName string) {
|
||||
// a directory converted to super large after accumulating members still has a legacy index
|
||||
if store.isSuperLargeDirectory(string(dirPath)) {
|
||||
|
||||
@@ -256,95 +256,6 @@ func TestListDirectoryEntriesDeletesLogicallyExpiredEntries(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDirectoryEntriesSkipsMissingAndExpiredMembersInOrder(t *testing.T) {
|
||||
for _, keyPrefix := range []string{"", "sw:"} {
|
||||
t.Run("keyPrefix="+keyPrefix, func(t *testing.T) {
|
||||
store, dir := newTestStore(t, keyPrefix)
|
||||
|
||||
insertTestEntry(t, store, dir.Child("a"), 0)
|
||||
insertTestEntry(t, store, dir.Child("b"), 0)
|
||||
insertLogicallyExpiredTestEntry(t, store, dir.Child("c"))
|
||||
insertTestEntry(t, store, dir.Child("d"), 0)
|
||||
|
||||
if err := store.Client.Del(context.Background(), store.getKey(string(dir.Child("b")))).Err(); err != nil {
|
||||
t.Fatalf("drop value key: %v", err)
|
||||
}
|
||||
|
||||
if names := listNames(t, store, dir); fmt.Sprint(names) != "[a d]" {
|
||||
t.Fatalf("listed %v, want [a d]", names)
|
||||
}
|
||||
|
||||
if members := indexMembers(t, store, dir); fmt.Sprint(members) != "[a d]" {
|
||||
t.Fatalf("directory index holds %v, want [a d]", members)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDirectoryEntriesAcrossFetchBatches(t *testing.T) {
|
||||
store, dir := newTestStore(t, "")
|
||||
|
||||
var want []string
|
||||
for i := 0; i < listFetchBatchSize+2; i++ {
|
||||
name := fmt.Sprintf("f%04d", i)
|
||||
insertTestEntry(t, store, dir.Child(name), 0)
|
||||
if i != listFetchBatchSize {
|
||||
want = append(want, name)
|
||||
}
|
||||
}
|
||||
|
||||
// the first member of the second batch
|
||||
missing := dir.Child(fmt.Sprintf("f%04d", listFetchBatchSize))
|
||||
if err := store.Client.Del(context.Background(), store.getKey(string(missing))).Err(); err != nil {
|
||||
t.Fatalf("drop value key: %v", err)
|
||||
}
|
||||
|
||||
var names []string
|
||||
if _, err := store.ListDirectoryEntries(context.Background(), dir, "", true, 1000, func(entry *filer.Entry) (bool, error) {
|
||||
names = append(names, entry.Name())
|
||||
return true, nil
|
||||
}); err != nil {
|
||||
t.Fatalf("ListDirectoryEntries %s: %v", dir, err)
|
||||
}
|
||||
if fmt.Sprint(names) != fmt.Sprint(want) {
|
||||
t.Fatalf("listed %v, want %v", names, want)
|
||||
}
|
||||
|
||||
if members := indexMembers(t, store, dir); fmt.Sprint(members) != fmt.Sprint(want) {
|
||||
t.Fatalf("directory index holds %v, want %v", members, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestListDirectoryEntriesStopsAtUnreadableMember(t *testing.T) {
|
||||
store, dir := newTestStore(t, "")
|
||||
ctx := context.Background()
|
||||
|
||||
insertTestEntry(t, store, dir.Child("a"), 0)
|
||||
insertTestEntry(t, store, dir.Child("b"), 0)
|
||||
insertTestEntry(t, store, dir.Child("c"), 0)
|
||||
|
||||
// a value of the wrong type makes its GET fail with something other than redis.Nil
|
||||
key := store.getKey(string(dir.Child("b")))
|
||||
if err := store.Client.Del(ctx, key).Err(); err != nil {
|
||||
t.Fatalf("drop value key: %v", err)
|
||||
}
|
||||
if err := store.Client.HSet(ctx, key, "f", "v").Err(); err != nil {
|
||||
t.Fatalf("plant hash: %v", err)
|
||||
}
|
||||
|
||||
var names []string
|
||||
lastFileName, err := store.ListDirectoryEntries(ctx, dir, "", true, 100, func(entry *filer.Entry) (bool, error) {
|
||||
names = append(names, entry.Name())
|
||||
return true, nil
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("ListDirectoryEntries returned no error for an unreadable member")
|
||||
}
|
||||
if fmt.Sprint(names) != "[a]" || lastFileName != "b" {
|
||||
t.Fatalf("listed %v up to %q, want [a] up to \"b\"", names, lastFileName)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeleteExpiredEntryKeepsRecreatedValue(t *testing.T) {
|
||||
store, dir := newTestStore(t, "")
|
||||
|
||||
|
||||
@@ -243,7 +243,7 @@ func (store *RocksDBStore) ListDirectoryPrefixedEntries(ctx context.Context, dir
|
||||
|
||||
directoryPrefix := genDirectoryKeyPrefix(dirPath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
|
||||
}
|
||||
|
||||
|
||||
@@ -246,7 +246,7 @@ func (store *TikvStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPat
|
||||
lastFileName := ""
|
||||
directoryPrefix := store.genDirectoryKeyPrefix(dirPath, prefix)
|
||||
lastFileStart := directoryPrefix
|
||||
if startFileName > prefix {
|
||||
if startFileName != "" {
|
||||
lastFileStart = store.genDirectoryKeyPrefix(dirPath, startFileName)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
// Package format maps the internal structure of container file formats onto
|
||||
// storage chunk boundaries.
|
||||
//
|
||||
// An adapter translates one format into three things the core understands: a
|
||||
// list of extent sizes, an alignment quantum, and an opaque payload. Adapters
|
||||
// never see chunks, file ids, or authorization; the core never learns what an
|
||||
// MPEG-TS packet or a parquet row group is.
|
||||
package format
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// LayoutKey is the filer Extended key holding the encoded Layout. The
|
||||
// x-seaweedfs- prefix keeps it out of HTTP response headers.
|
||||
const LayoutKey = "x-seaweedfs-format-layout"
|
||||
|
||||
// ViewParam is the query parameter selecting a format view on GET requests.
|
||||
// Adapters rendering self-referential URLs must use it.
|
||||
const ViewParam = "format.view"
|
||||
|
||||
// ErrNoSuchView reports that a view request addresses nothing servable; the
|
||||
// server answers 404.
|
||||
var ErrNoSuchView = errors.New("no such view")
|
||||
|
||||
// Layout describes how a file's structure maps to byte extents.
|
||||
type Layout struct {
|
||||
Format string // adapter name
|
||||
ExtentSizes []int64 // extent lengths in file order; they sum to the file size
|
||||
Align int64 // quantum for cutting inside an oversized extent; 1 cuts anywhere
|
||||
Payload []byte // adapter-owned metadata, opaque to the core
|
||||
}
|
||||
|
||||
// Hint carries the cheap identification signals available to Sniff.
|
||||
type Hint struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Size int64
|
||||
Head []byte
|
||||
Tail []byte
|
||||
}
|
||||
|
||||
// Format is the mandatory adapter identity. Capabilities beyond it are
|
||||
// discovered by type assertion.
|
||||
type Format interface {
|
||||
Name() string
|
||||
}
|
||||
|
||||
// Sniffer cheaply recognizes the format from identification signals. Repack
|
||||
// gates on it before parsing, and policy-driven detection will rely on it;
|
||||
// ingest-only adapters, whose files carry their layout from birth, skip it.
|
||||
type Sniffer interface {
|
||||
Sniff(h Hint) bool
|
||||
}
|
||||
|
||||
// Indexer derives a Layout from the complete stored bytes.
|
||||
type Indexer interface {
|
||||
Index(ctx context.Context, r io.ReaderAt, size int64) (*Layout, error)
|
||||
}
|
||||
|
||||
// SidecarIndexer derives a Layout from an external index document supplied at
|
||||
// ingest, before the media bytes arrive.
|
||||
type SidecarIndexer interface {
|
||||
IndexSidecar(sidecar []byte) (*Layout, error)
|
||||
}
|
||||
|
||||
// Object is everything a Viewer may know about the file it serves.
|
||||
type Object struct {
|
||||
Name string
|
||||
Size int64
|
||||
Layout *Layout
|
||||
}
|
||||
|
||||
// ViewRequest carries the request parameters of a ?view= request.
|
||||
type ViewRequest struct {
|
||||
Query url.Values
|
||||
}
|
||||
|
||||
// ViewPlan tells the server what to serve. The server executes it on the
|
||||
// normal streaming path; adapters stay pure functions of request and layout.
|
||||
type ViewPlan struct {
|
||||
ContentType string
|
||||
Body []byte // rendered document; when nil, stream Extent instead
|
||||
Extent int
|
||||
}
|
||||
|
||||
// Viewer answers ?view= requests.
|
||||
type Viewer interface {
|
||||
View(req ViewRequest, obj Object) (*ViewPlan, error)
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Package formattest is the conformance kit format adapters must pass:
|
||||
// indexers parse attacker-controlled bytes inside a storage daemon, so they
|
||||
// must never panic and every layout they accept must validate.
|
||||
package formattest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
)
|
||||
|
||||
// IndexTruncations feeds progressively truncated copies of a valid file to the
|
||||
// indexer. Any outcome is acceptable except a panic or an invalid layout.
|
||||
func IndexTruncations(t *testing.T, indexer format.Indexer, data []byte) {
|
||||
t.Helper()
|
||||
for i := 0; i <= 16; i++ {
|
||||
size := int64(len(data) * i / 16)
|
||||
layout, err := indexer.Index(context.Background(), bytes.NewReader(data[:size]), size)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if validateErr := layout.Validate(size); validateErr != nil {
|
||||
t.Fatalf("Index() at %d/%d bytes returned an invalid layout: %v", size, len(data), validateErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SidecarTruncations does the same for sidecar index documents.
|
||||
func SidecarTruncations(t *testing.T, indexer format.SidecarIndexer, sidecar []byte) {
|
||||
t.Helper()
|
||||
for i := 0; i <= 16; i++ {
|
||||
layout, err := indexer.IndexSidecar(sidecar[:len(sidecar)*i/16])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if validateErr := layout.Validate(-1); validateErr != nil {
|
||||
t.Fatalf("IndexSidecar() at %d/16 returned an invalid layout: %v", i, validateErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// EncodeRoundTrip checks that a layout survives the persistence codec.
|
||||
func EncodeRoundTrip(t *testing.T, layout *format.Layout) {
|
||||
t.Helper()
|
||||
encoded, err := layout.Encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v", err)
|
||||
}
|
||||
decoded, err := format.DecodeLayout(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeLayout() error = %v", err)
|
||||
}
|
||||
if decoded.Format != layout.Format || decoded.Align != layout.Align ||
|
||||
len(decoded.ExtentSizes) != len(layout.ExtentSizes) || !bytes.Equal(decoded.Payload, layout.Payload) {
|
||||
t.Fatalf("decoded layout %+v differs from %+v", decoded, layout)
|
||||
}
|
||||
for i := range layout.ExtentSizes {
|
||||
if decoded.ExtentSizes[i] != layout.ExtentSizes[i] {
|
||||
t.Fatalf("extent %d = %d, want %d", i, decoded.ExtentSizes[i], layout.ExtentSizes[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,288 @@
|
||||
// Package hlsts adapts single-file HLS MPEG-TS VOD assets: the ingest sidecar
|
||||
// is an FFmpeg-style EXT-X-BYTERANGE media playlist, extents are its segments,
|
||||
// and the view serves a rewritten playlist with plain numbered segment URLs.
|
||||
package hlsts
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
)
|
||||
|
||||
const (
|
||||
FormatName = "hls-ts"
|
||||
// TSPacketSize is the fixed MPEG-TS packet size; segment boundaries and
|
||||
// interior chunk cuts land on packet boundaries.
|
||||
TSPacketSize = 188
|
||||
|
||||
tsSyncByte = 0x47
|
||||
|
||||
PlaylistContentType = "application/vnd.apple.mpegurl"
|
||||
MediaContentType = "video/MP2T"
|
||||
)
|
||||
|
||||
func init() {
|
||||
format.Register(Adapter{})
|
||||
}
|
||||
|
||||
type Adapter struct{}
|
||||
|
||||
var (
|
||||
_ format.Sniffer = Adapter{}
|
||||
_ format.SidecarIndexer = Adapter{}
|
||||
_ format.Viewer = Adapter{}
|
||||
)
|
||||
|
||||
func (Adapter) Name() string { return FormatName }
|
||||
|
||||
func (Adapter) Sniff(h format.Hint) bool {
|
||||
if len(h.Head) > TSPacketSize {
|
||||
return h.Head[0] == tsSyncByte && h.Head[TSPacketSize] == tsSyncByte
|
||||
}
|
||||
return len(h.Head) > 0 && h.Head[0] == tsSyncByte
|
||||
}
|
||||
|
||||
// playlistInfo is the adapter payload: what the generated playback playlist
|
||||
// needs beyond the extent sizes.
|
||||
type playlistInfo struct {
|
||||
TargetDuration int64
|
||||
MediaSequence int64
|
||||
DurationsMs []int64
|
||||
}
|
||||
|
||||
func (p *playlistInfo) encode() []byte {
|
||||
out := binary.AppendUvarint(nil, uint64(p.TargetDuration))
|
||||
out = binary.AppendUvarint(out, uint64(p.MediaSequence))
|
||||
for _, durationMs := range p.DurationsMs {
|
||||
out = binary.AppendUvarint(out, uint64(durationMs))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func decodePlaylistInfo(payload []byte, extentCount int) (*playlistInfo, error) {
|
||||
reader := bytes.NewReader(payload)
|
||||
target, err := binary.ReadUvarint(reader)
|
||||
if err != nil || target == 0 || target > math.MaxInt32 {
|
||||
return nil, fmt.Errorf("invalid hls-ts target duration")
|
||||
}
|
||||
// mirror the ingest bound: the last segment number is sequence+count-1
|
||||
sequence, err := binary.ReadUvarint(reader)
|
||||
if err != nil || sequence > math.MaxInt64-uint64(extentCount-1) {
|
||||
return nil, fmt.Errorf("invalid hls-ts media sequence")
|
||||
}
|
||||
info := &playlistInfo{TargetDuration: int64(target), MediaSequence: int64(sequence), DurationsMs: make([]int64, extentCount)}
|
||||
for i := range info.DurationsMs {
|
||||
durationMs, err := binary.ReadUvarint(reader)
|
||||
if err != nil || durationMs == 0 || durationMs > math.MaxInt32 {
|
||||
return nil, fmt.Errorf("invalid hls-ts segment %d duration", i)
|
||||
}
|
||||
info.DurationsMs[i] = int64(durationMs)
|
||||
}
|
||||
if reader.Len() != 0 {
|
||||
return nil, fmt.Errorf("hls-ts payload has trailing bytes")
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// IndexSidecar parses a VOD media playlist whose segments reference one shared
|
||||
// media URI through EXT-X-BYTERANGE. Playlist state the generated playback
|
||||
// playlist cannot reproduce is rejected.
|
||||
func (Adapter) IndexSidecar(sidecar []byte) (*format.Layout, error) {
|
||||
scanner := bufio.NewScanner(bytes.NewReader(sidecar))
|
||||
scanner.Buffer(make([]byte, 64*1024), 1<<20)
|
||||
|
||||
info := &playlistInfo{}
|
||||
var sizes []int64
|
||||
var pendingDuration int64 // ms; 0 = no EXTINF pending
|
||||
var pendingSize, pendingOffset int64
|
||||
var havePendingRange bool
|
||||
var expectedOffset int64
|
||||
var mediaURI string
|
||||
var sawHeader, sawEndList bool
|
||||
var maxDurationMs int64
|
||||
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
switch {
|
||||
case line == "":
|
||||
case line == "#EXTM3U":
|
||||
sawHeader = true
|
||||
case line == "#EXT-X-ENDLIST":
|
||||
sawEndList = true
|
||||
case strings.HasPrefix(line, "#EXT-X-KEY:"),
|
||||
line == "#EXT-X-DISCONTINUITY",
|
||||
strings.HasPrefix(line, "#EXT-X-DISCONTINUITY-SEQUENCE:"),
|
||||
strings.HasPrefix(line, "#EXT-X-MAP:"),
|
||||
line == "#EXT-X-GAP",
|
||||
line == "#EXT-X-I-FRAMES-ONLY":
|
||||
return nil, fmt.Errorf("%s is not supported by hls-ts ingest", strings.SplitN(line, ":", 2)[0])
|
||||
case strings.HasPrefix(line, "#EXT-X-TARGETDURATION:"):
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, "#EXT-X-TARGETDURATION:"))
|
||||
target, err := strconv.ParseInt(value, 10, 32)
|
||||
if err != nil || target <= 0 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-TARGETDURATION %q", value)
|
||||
}
|
||||
info.TargetDuration = target
|
||||
case strings.HasPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"):
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, "#EXT-X-MEDIA-SEQUENCE:"))
|
||||
sequence, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || sequence < 0 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-MEDIA-SEQUENCE %q", value)
|
||||
}
|
||||
info.MediaSequence = sequence
|
||||
case strings.HasPrefix(line, "#EXTINF:"):
|
||||
if pendingDuration != 0 {
|
||||
return nil, errors.New("EXTINF without a media URI for the previous segment")
|
||||
}
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, "#EXTINF:"))
|
||||
if comma := strings.IndexByte(value, ','); comma >= 0 {
|
||||
value = value[:comma]
|
||||
}
|
||||
seconds, err := strconv.ParseFloat(value, 64)
|
||||
if err != nil || seconds <= 0 || math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds > math.MaxInt32/1000 {
|
||||
return nil, fmt.Errorf("invalid EXTINF duration %q", value)
|
||||
}
|
||||
pendingDuration = int64(math.Round(seconds * 1000))
|
||||
if pendingDuration == 0 {
|
||||
pendingDuration = 1
|
||||
}
|
||||
case strings.HasPrefix(line, "#EXT-X-BYTERANGE:"):
|
||||
if pendingDuration == 0 {
|
||||
return nil, errors.New("EXT-X-BYTERANGE without a preceding EXTINF")
|
||||
}
|
||||
value := strings.TrimSpace(strings.TrimPrefix(line, "#EXT-X-BYTERANGE:"))
|
||||
lengthText, offsetText, hasOffset := strings.Cut(value, "@")
|
||||
length, err := strconv.ParseInt(lengthText, 10, 64)
|
||||
if err != nil || length <= 0 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE length %q", value)
|
||||
}
|
||||
offset := expectedOffset
|
||||
if hasOffset {
|
||||
if offset, err = strconv.ParseInt(offsetText, 10, 64); err != nil || offset < 0 {
|
||||
return nil, fmt.Errorf("invalid EXT-X-BYTERANGE offset %q", value)
|
||||
}
|
||||
}
|
||||
pendingSize, pendingOffset, havePendingRange = length, offset, true
|
||||
case strings.HasPrefix(line, "#"):
|
||||
// other tags carry no state the generated playlist must keep
|
||||
default:
|
||||
if pendingDuration == 0 || !havePendingRange {
|
||||
return nil, fmt.Errorf("media URI %q without EXTINF and EXT-X-BYTERANGE", line)
|
||||
}
|
||||
if mediaURI == "" {
|
||||
mediaURI = line
|
||||
} else if mediaURI != line {
|
||||
return nil, errors.New("hls-ts ingest requires one shared media URI")
|
||||
}
|
||||
if pendingOffset != expectedOffset {
|
||||
return nil, fmt.Errorf("non-contiguous byte range at segment %d: offset %d, expected %d", len(sizes), pendingOffset, expectedOffset)
|
||||
}
|
||||
if pendingSize%TSPacketSize != 0 {
|
||||
return nil, fmt.Errorf("segment %d size %d is not a multiple of the %d-byte TS packet", len(sizes), pendingSize, TSPacketSize)
|
||||
}
|
||||
if pendingSize > math.MaxInt64-expectedOffset {
|
||||
return nil, fmt.Errorf("byte range at segment %d overflows", len(sizes))
|
||||
}
|
||||
if len(sizes) >= format.MaxExtentCount {
|
||||
return nil, fmt.Errorf("playlist has more than %d segments", format.MaxExtentCount)
|
||||
}
|
||||
sizes = append(sizes, pendingSize)
|
||||
info.DurationsMs = append(info.DurationsMs, pendingDuration)
|
||||
if pendingDuration > maxDurationMs {
|
||||
maxDurationMs = pendingDuration
|
||||
}
|
||||
expectedOffset += pendingSize
|
||||
pendingDuration, havePendingRange = 0, false
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("read playlist: %w", err)
|
||||
}
|
||||
if !sawHeader {
|
||||
return nil, errors.New("playlist is missing EXTM3U")
|
||||
}
|
||||
if pendingDuration != 0 || havePendingRange {
|
||||
return nil, errors.New("playlist ended with an incomplete media segment")
|
||||
}
|
||||
if len(sizes) == 0 {
|
||||
return nil, errors.New("playlist has no media segments")
|
||||
}
|
||||
if !sawEndList {
|
||||
return nil, errors.New("only VOD playlists with EXT-X-ENDLIST are supported")
|
||||
}
|
||||
if info.MediaSequence > math.MaxInt64-int64(len(sizes)-1) {
|
||||
return nil, errors.New("EXT-X-MEDIA-SEQUENCE overflows segment numbering")
|
||||
}
|
||||
|
||||
// RFC 8216: EXT-X-TARGETDURATION must be at least each segment duration
|
||||
// rounded to the nearest integer.
|
||||
minimumTarget := (maxDurationMs + 500) / 1000
|
||||
if minimumTarget < 1 {
|
||||
minimumTarget = 1
|
||||
}
|
||||
if info.TargetDuration == 0 {
|
||||
info.TargetDuration = minimumTarget
|
||||
} else if info.TargetDuration < minimumTarget {
|
||||
return nil, fmt.Errorf("EXT-X-TARGETDURATION %d is smaller than the longest segment duration %d", info.TargetDuration, minimumTarget)
|
||||
}
|
||||
|
||||
layout := &format.Layout{
|
||||
Format: FormatName,
|
||||
ExtentSizes: sizes,
|
||||
Align: TSPacketSize,
|
||||
Payload: info.encode(),
|
||||
}
|
||||
// valid by construction, but enforce the formattest invariant explicitly
|
||||
if err := layout.Validate(-1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return layout, nil
|
||||
}
|
||||
|
||||
// View serves the generated playlist, or maps ?seq=N to its extent.
|
||||
func (Adapter) View(req format.ViewRequest, obj format.Object) (*format.ViewPlan, error) {
|
||||
info, err := decodePlaylistInfo(obj.Layout.Payload, len(obj.Layout.ExtentSizes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sequenceText := req.Query.Get("seq")
|
||||
if sequenceText == "" {
|
||||
return &format.ViewPlan{ContentType: PlaylistContentType, Body: renderPlaylist(obj.Name, info, req.Query.Get("jwt"))}, nil
|
||||
}
|
||||
sequence, err := strconv.ParseInt(sequenceText, 10, 64)
|
||||
if err != nil || sequence < info.MediaSequence {
|
||||
return nil, format.ErrNoSuchView
|
||||
}
|
||||
index := sequence - info.MediaSequence
|
||||
if index >= int64(len(obj.Layout.ExtentSizes)) {
|
||||
return nil, format.ErrNoSuchView
|
||||
}
|
||||
return &format.ViewPlan{ContentType: MediaContentType, Extent: int(index)}, nil
|
||||
}
|
||||
|
||||
func renderPlaylist(name string, info *playlistInfo, jwt string) []byte {
|
||||
var out strings.Builder
|
||||
out.WriteString("#EXTM3U\n#EXT-X-VERSION:3\n")
|
||||
fmt.Fprintf(&out, "#EXT-X-TARGETDURATION:%d\n", info.TargetDuration)
|
||||
fmt.Fprintf(&out, "#EXT-X-MEDIA-SEQUENCE:%d\n", info.MediaSequence)
|
||||
out.WriteString("#EXT-X-PLAYLIST-TYPE:VOD\n")
|
||||
escapedName := url.PathEscape(name)
|
||||
credential := ""
|
||||
if jwt != "" {
|
||||
credential = "&jwt=" + url.QueryEscape(jwt)
|
||||
}
|
||||
for i, durationMs := range info.DurationsMs {
|
||||
fmt.Fprintf(&out, "#EXTINF:%.3f,\n", float64(durationMs)/1000)
|
||||
fmt.Fprintf(&out, "./%s?%s=%s&seq=%d%s\n", escapedName, format.ViewParam, FormatName, int64(i)+info.MediaSequence, credential)
|
||||
}
|
||||
out.WriteString("#EXT-X-ENDLIST\n")
|
||||
return []byte(out.String())
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package hlsts
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format/formattest"
|
||||
)
|
||||
|
||||
const ffmpegPlaylist = `#EXTM3U
|
||||
#EXT-X-VERSION:4
|
||||
#EXT-X-TARGETDURATION:6
|
||||
#EXT-X-MEDIA-SEQUENCE:0
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXTINF:6.000000,
|
||||
#EXT-X-BYTERANGE:1128@0
|
||||
video.ts
|
||||
#EXTINF:6.000000,
|
||||
#EXT-X-BYTERANGE:940
|
||||
video.ts
|
||||
#EXTINF:2.500000,
|
||||
#EXT-X-BYTERANGE:376@2068
|
||||
video.ts
|
||||
#EXT-X-ENDLIST
|
||||
`
|
||||
|
||||
func TestIndexSidecar(t *testing.T) {
|
||||
layout, err := Adapter{}.IndexSidecar([]byte(ffmpegPlaylist))
|
||||
if err != nil {
|
||||
t.Fatalf("IndexSidecar() error = %v", err)
|
||||
}
|
||||
wantSizes := []int64{1128, 940, 376}
|
||||
if len(layout.ExtentSizes) != len(wantSizes) {
|
||||
t.Fatalf("extents = %v, want %v", layout.ExtentSizes, wantSizes)
|
||||
}
|
||||
for i := range wantSizes {
|
||||
if layout.ExtentSizes[i] != wantSizes[i] {
|
||||
t.Fatalf("extent %d = %d, want %d", i, layout.ExtentSizes[i], wantSizes[i])
|
||||
}
|
||||
}
|
||||
if layout.Align != TSPacketSize || layout.Format != FormatName {
|
||||
t.Fatalf("layout = %+v", layout)
|
||||
}
|
||||
if err := layout.Validate(1128 + 940 + 376); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
formattest.EncodeRoundTrip(t, layout)
|
||||
}
|
||||
|
||||
func TestIndexSidecarDefaultsTargetDuration(t *testing.T) {
|
||||
playlist := "#EXTM3U\n#EXTINF:5.6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n"
|
||||
layout, err := Adapter{}.IndexSidecar([]byte(playlist))
|
||||
if err != nil {
|
||||
t.Fatalf("IndexSidecar() error = %v", err)
|
||||
}
|
||||
info, err := decodePlaylistInfo(layout.Payload, len(layout.ExtentSizes))
|
||||
if err != nil {
|
||||
t.Fatalf("decodePlaylistInfo() error = %v", err)
|
||||
}
|
||||
if info.TargetDuration != 6 {
|
||||
t.Fatalf("TargetDuration = %d, want 6", info.TargetDuration)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexSidecarRejections(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
playlist string
|
||||
wantErr string
|
||||
}{
|
||||
{"missing header", "#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXTM3U"},
|
||||
{"missing endlist", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n", "EXT-X-ENDLIST"},
|
||||
{"no segments", "#EXTM3U\n#EXT-X-ENDLIST\n", "no media segments"},
|
||||
{"encryption", "#EXTM3U\n#EXT-X-KEY:METHOD=AES-128\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXT-X-KEY"},
|
||||
{"discontinuity", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-DISCONTINUITY\n#EXT-X-ENDLIST\n", "EXT-X-DISCONTINUITY"},
|
||||
{"map", "#EXTM3U\n#EXT-X-MAP:URI=\"init.mp4\"\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXT-X-MAP"},
|
||||
{"gap", "#EXTM3U\n#EXT-X-GAP\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXT-X-GAP"},
|
||||
{"iframes only", "#EXTM3U\n#EXT-X-I-FRAMES-ONLY\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXT-X-I-FRAMES-ONLY"},
|
||||
{"no byterange", "#EXTM3U\n#EXTINF:6,\nv.ts\n#EXT-X-ENDLIST\n", "EXT-X-BYTERANGE"},
|
||||
{"gap in ranges", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@376\nv.ts\n#EXT-X-ENDLIST\n", "non-contiguous"},
|
||||
{"unaligned size", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:100@0\nv.ts\n#EXT-X-ENDLIST\n", "TS packet"},
|
||||
{"two media files", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\na.ts\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@188\nb.ts\n#EXT-X-ENDLIST\n", "one shared media URI"},
|
||||
{"target too small", "#EXTM3U\n#EXT-X-TARGETDURATION:2\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "smaller than"},
|
||||
{"zero duration", "#EXTM3U\n#EXTINF:0,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXT-X-ENDLIST\n", "EXTINF"},
|
||||
{"dangling extinf", "#EXTM3U\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXTINF:6,\n#EXT-X-ENDLIST\n", "incomplete"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
_, err := Adapter{}.IndexSidecar([]byte(test.playlist))
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("%s: error = %v, want %q", test.name, err, test.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSidecarTruncations(t *testing.T) {
|
||||
formattest.SidecarTruncations(t, Adapter{}, []byte(ffmpegPlaylist))
|
||||
}
|
||||
|
||||
func viewObject(t *testing.T) format.Object {
|
||||
t.Helper()
|
||||
layout, err := Adapter{}.IndexSidecar([]byte(ffmpegPlaylist))
|
||||
if err != nil {
|
||||
t.Fatalf("IndexSidecar() error = %v", err)
|
||||
}
|
||||
return format.Object{Name: "movie.ts", Size: layout.TotalSize(), Layout: layout}
|
||||
}
|
||||
|
||||
func TestViewPlaylist(t *testing.T) {
|
||||
plan, err := Adapter{}.View(format.ViewRequest{Query: url.Values{}}, viewObject(t))
|
||||
if err != nil {
|
||||
t.Fatalf("View() error = %v", err)
|
||||
}
|
||||
if plan.ContentType != PlaylistContentType {
|
||||
t.Fatalf("ContentType = %q", plan.ContentType)
|
||||
}
|
||||
want := `#EXTM3U
|
||||
#EXT-X-VERSION:3
|
||||
#EXT-X-TARGETDURATION:6
|
||||
#EXT-X-MEDIA-SEQUENCE:0
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXTINF:6.000,
|
||||
./movie.ts?format.view=hls-ts&seq=0
|
||||
#EXTINF:6.000,
|
||||
./movie.ts?format.view=hls-ts&seq=1
|
||||
#EXTINF:2.500,
|
||||
./movie.ts?format.view=hls-ts&seq=2
|
||||
#EXT-X-ENDLIST
|
||||
`
|
||||
if string(plan.Body) != want {
|
||||
t.Fatalf("playlist = %q, want %q", plan.Body, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewPlaylistCarriesJwt(t *testing.T) {
|
||||
plan, err := Adapter{}.View(format.ViewRequest{Query: url.Values{"jwt": {"a.b+c/d="}}}, viewObject(t))
|
||||
if err != nil {
|
||||
t.Fatalf("View() error = %v", err)
|
||||
}
|
||||
for _, line := range strings.Split(string(plan.Body), "\n") {
|
||||
if strings.HasSuffix(line, ".ts") || strings.Contains(line, "seq=") {
|
||||
if !strings.Contains(line, "jwt=a.b%2Bc%2Fd%3D") {
|
||||
t.Fatalf("segment URL dropped jwt: %q", line)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewSegment(t *testing.T) {
|
||||
obj := viewObject(t)
|
||||
plan, err := Adapter{}.View(format.ViewRequest{Query: url.Values{"seq": {"1"}}}, obj)
|
||||
if err != nil {
|
||||
t.Fatalf("View() error = %v", err)
|
||||
}
|
||||
if plan.Body != nil || plan.Extent != 1 || plan.ContentType != MediaContentType {
|
||||
t.Fatalf("plan = %+v", plan)
|
||||
}
|
||||
for _, bad := range []string{"3", "-1", "x", "9999999999999999999"} {
|
||||
if _, err := (Adapter{}).View(format.ViewRequest{Query: url.Values{"seq": {bad}}}, obj); err != format.ErrNoSuchView {
|
||||
t.Fatalf("seq %q: error = %v, want ErrNoSuchView", bad, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestViewSegmentHonorsMediaSequence(t *testing.T) {
|
||||
playlist := "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:10\n#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXTINF:6,\n#EXT-X-BYTERANGE:376\nv.ts\n#EXT-X-ENDLIST\n"
|
||||
layout, err := Adapter{}.IndexSidecar([]byte(playlist))
|
||||
if err != nil {
|
||||
t.Fatalf("IndexSidecar() error = %v", err)
|
||||
}
|
||||
obj := format.Object{Name: "v.ts", Size: layout.TotalSize(), Layout: layout}
|
||||
plan, err := Adapter{}.View(format.ViewRequest{Query: url.Values{"seq": {"11"}}}, obj)
|
||||
if err != nil {
|
||||
t.Fatalf("View() error = %v", err)
|
||||
}
|
||||
if plan.Extent != 1 {
|
||||
t.Fatalf("Extent = %d, want 1", plan.Extent)
|
||||
}
|
||||
if _, err := (Adapter{}).View(format.ViewRequest{Query: url.Values{"seq": {"9"}}}, obj); err != format.ErrNoSuchView {
|
||||
t.Fatalf("seq below media sequence: error = %v, want ErrNoSuchView", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSniff(t *testing.T) {
|
||||
head := make([]byte, 400)
|
||||
head[0], head[TSPacketSize] = tsSyncByte, tsSyncByte
|
||||
if !(Adapter{}).Sniff(format.Hint{Head: head}) {
|
||||
t.Fatalf("Sniff() rejected TS head")
|
||||
}
|
||||
head[TSPacketSize] = 0
|
||||
if (Adapter{}).Sniff(format.Hint{Head: head}) {
|
||||
t.Fatalf("Sniff() accepted non-TS head")
|
||||
}
|
||||
}
|
||||
|
||||
// The ingest bound admits mediaSequence = MaxInt64-(count-1); the payload
|
||||
// decoder must accept the same boundary or every view of such an asset fails.
|
||||
func TestMediaSequenceBoundaryRoundTrip(t *testing.T) {
|
||||
playlist := "#EXTM3U\n#EXT-X-MEDIA-SEQUENCE:9223372036854775806\n" +
|
||||
"#EXTINF:6,\n#EXT-X-BYTERANGE:188@0\nv.ts\n#EXTINF:6,\n#EXT-X-BYTERANGE:188\nv.ts\n#EXT-X-ENDLIST\n"
|
||||
layout, err := Adapter{}.IndexSidecar([]byte(playlist))
|
||||
if err != nil {
|
||||
t.Fatalf("IndexSidecar() error = %v", err)
|
||||
}
|
||||
obj := format.Object{Name: "v.ts", Size: layout.TotalSize(), Layout: layout}
|
||||
if _, err := (Adapter{}).View(format.ViewRequest{Query: url.Values{}}, obj); err != nil {
|
||||
t.Fatalf("playlist view error = %v", err)
|
||||
}
|
||||
plan, err := Adapter{}.View(format.ViewRequest{Query: url.Values{"seq": {"9223372036854775807"}}}, obj)
|
||||
if err != nil {
|
||||
t.Fatalf("last segment view error = %v", err)
|
||||
}
|
||||
if plan.Extent != 1 {
|
||||
t.Fatalf("Extent = %d, want 1", plan.Extent)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
)
|
||||
|
||||
const (
|
||||
layoutVersion = 1
|
||||
|
||||
// MaxExtentCount bounds decoded layouts; it also bounds the chunk count a
|
||||
// layout can force on an entry.
|
||||
MaxExtentCount = 1 << 20
|
||||
// MaxPayloadBytes bounds the adapter payload carried in entry metadata.
|
||||
MaxPayloadBytes = 16 << 20
|
||||
// MaxFormatNameBytes bounds the encoded adapter name.
|
||||
MaxFormatNameBytes = 256
|
||||
)
|
||||
|
||||
// TotalSize returns the file size the layout describes.
|
||||
func (l *Layout) TotalSize() int64 {
|
||||
var total int64
|
||||
for _, size := range l.ExtentSizes {
|
||||
total += size
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// ExtentRange returns the byte range of extent i.
|
||||
func (l *Layout) ExtentRange(i int) (offset, size int64, ok bool) {
|
||||
if i < 0 || i >= len(l.ExtentSizes) {
|
||||
return 0, 0, false
|
||||
}
|
||||
for _, extentSize := range l.ExtentSizes[:i] {
|
||||
offset += extentSize
|
||||
}
|
||||
return offset, l.ExtentSizes[i], true
|
||||
}
|
||||
|
||||
// Validate checks layout consistency. A negative fileSize skips the total size
|
||||
// check.
|
||||
func (l *Layout) Validate(fileSize int64) error {
|
||||
if l.Format == "" {
|
||||
return fmt.Errorf("layout has no format name")
|
||||
}
|
||||
if len(l.Format) > MaxFormatNameBytes {
|
||||
return fmt.Errorf("layout format name is too long: %d bytes", len(l.Format))
|
||||
}
|
||||
if l.Align < 1 {
|
||||
return fmt.Errorf("layout align %d is invalid", l.Align)
|
||||
}
|
||||
if len(l.ExtentSizes) == 0 {
|
||||
return fmt.Errorf("layout has no extents")
|
||||
}
|
||||
if len(l.ExtentSizes) > MaxExtentCount {
|
||||
return fmt.Errorf("layout has too many extents: %d", len(l.ExtentSizes))
|
||||
}
|
||||
if len(l.Payload) > MaxPayloadBytes {
|
||||
return fmt.Errorf("layout payload is too large: %d bytes", len(l.Payload))
|
||||
}
|
||||
var total int64
|
||||
for i, size := range l.ExtentSizes {
|
||||
if size <= 0 {
|
||||
return fmt.Errorf("extent %d has invalid size %d", i, size)
|
||||
}
|
||||
if size > math.MaxInt64-total {
|
||||
return fmt.Errorf("extent %d overflows the file size", i)
|
||||
}
|
||||
total += size
|
||||
}
|
||||
if fileSize >= 0 && total != fileSize {
|
||||
return fmt.Errorf("layout describes %d bytes but the file has %d", total, fileSize)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode serializes the layout for the LayoutKey extended attribute.
|
||||
func (l *Layout) Encode() ([]byte, error) {
|
||||
if err := l.Validate(-1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []byte{layoutVersion}
|
||||
out = binary.AppendUvarint(out, uint64(len(l.Format)))
|
||||
out = append(out, l.Format...)
|
||||
out = binary.AppendUvarint(out, uint64(l.Align))
|
||||
out = binary.AppendUvarint(out, uint64(len(l.ExtentSizes)))
|
||||
for _, size := range l.ExtentSizes {
|
||||
out = binary.AppendUvarint(out, uint64(size))
|
||||
}
|
||||
out = binary.AppendUvarint(out, uint64(len(l.Payload)))
|
||||
out = append(out, l.Payload...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// DecodeLayout parses an encoded layout and validates it.
|
||||
func DecodeLayout(data []byte) (*Layout, error) {
|
||||
reader := bytes.NewReader(data)
|
||||
version, err := reader.ReadByte()
|
||||
if err != nil || version != layoutVersion {
|
||||
return nil, fmt.Errorf("unsupported layout version")
|
||||
}
|
||||
name, err := readUvarintBytes(reader, MaxFormatNameBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read layout format: %w", err)
|
||||
}
|
||||
align, err := binary.ReadUvarint(reader)
|
||||
if err != nil || align > math.MaxInt64 {
|
||||
return nil, fmt.Errorf("read layout align: invalid")
|
||||
}
|
||||
count, err := binary.ReadUvarint(reader)
|
||||
if err != nil || count > MaxExtentCount {
|
||||
return nil, fmt.Errorf("read layout extent count: invalid")
|
||||
}
|
||||
sizes := make([]int64, count)
|
||||
for i := range sizes {
|
||||
size, err := binary.ReadUvarint(reader)
|
||||
if err != nil || size > math.MaxInt64 {
|
||||
return nil, fmt.Errorf("read extent %d size: invalid", i)
|
||||
}
|
||||
sizes[i] = int64(size)
|
||||
}
|
||||
payload, err := readUvarintBytes(reader, MaxPayloadBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read layout payload: %w", err)
|
||||
}
|
||||
if reader.Len() != 0 {
|
||||
return nil, fmt.Errorf("layout has %d trailing bytes", reader.Len())
|
||||
}
|
||||
layout := &Layout{Format: string(name), ExtentSizes: sizes, Align: int64(align), Payload: payload}
|
||||
if err := layout.Validate(-1); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return layout, nil
|
||||
}
|
||||
|
||||
func readUvarintBytes(reader *bytes.Reader, limit uint64) ([]byte, error) {
|
||||
length, err := binary.ReadUvarint(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if length > limit || length > uint64(reader.Len()) {
|
||||
return nil, fmt.Errorf("length %d is out of bounds", length)
|
||||
}
|
||||
if length == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data := make([]byte, length)
|
||||
if _, err := reader.Read(data); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// Cutter yields upload chunk boundaries: every extent boundary, plus
|
||||
// align-quantized cuts inside extents larger than maxChunkSize. A non-positive
|
||||
// maxChunkSize keeps each extent in one chunk. Interior cuts are computed
|
||||
// lazily, so memory stays bounded by the extent count no matter how large an
|
||||
// untrusted layout declares its extents to be.
|
||||
type Cutter struct {
|
||||
starts []int64 // extent start offsets, ascending
|
||||
total int64
|
||||
quantum int64 // interior cut spacing; 0 = one chunk per extent
|
||||
}
|
||||
|
||||
func (l *Layout) Cutter(maxChunkSize int64) *Cutter {
|
||||
quantum := maxChunkSize
|
||||
if quantum > 0 && l.Align > 1 {
|
||||
quantum -= quantum % l.Align
|
||||
if quantum <= 0 {
|
||||
// An align larger than the chunk limit still cuts on whole atoms.
|
||||
quantum = l.Align
|
||||
}
|
||||
}
|
||||
starts := make([]int64, len(l.ExtentSizes))
|
||||
var offset int64
|
||||
for i, size := range l.ExtentSizes {
|
||||
starts[i] = offset
|
||||
offset += size
|
||||
}
|
||||
return &Cutter{starts: starts, total: offset, quantum: quantum}
|
||||
}
|
||||
|
||||
// NextChunkSize returns the size of the chunk starting at offset, or 0 past
|
||||
// the end. It satisfies the filer upload loop's ChunkBoundaries interface.
|
||||
func (c *Cutter) NextChunkSize(offset int64) int64 {
|
||||
if offset < 0 || offset >= c.total {
|
||||
return 0
|
||||
}
|
||||
i := sort.Search(len(c.starts), func(i int) bool { return c.starts[i] > offset }) - 1
|
||||
end := c.total
|
||||
if i+1 < len(c.starts) {
|
||||
end = c.starts[i+1]
|
||||
}
|
||||
remaining := end - offset
|
||||
if c.quantum > 0 {
|
||||
if step := c.quantum - (offset-c.starts[i])%c.quantum; step < remaining {
|
||||
return step
|
||||
}
|
||||
}
|
||||
return remaining
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
package format
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestLayoutEncodeDecodeRoundTrip(t *testing.T) {
|
||||
layout := &Layout{
|
||||
Format: "hls-ts",
|
||||
ExtentSizes: []int64{188 * 3, 188 * 2, 188 * 7},
|
||||
Align: 188,
|
||||
Payload: []byte{1, 2, 3},
|
||||
}
|
||||
encoded, err := layout.Encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v", err)
|
||||
}
|
||||
decoded, err := DecodeLayout(encoded)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeLayout() error = %v", err)
|
||||
}
|
||||
if decoded.Format != layout.Format || decoded.Align != layout.Align {
|
||||
t.Fatalf("decoded = %+v, want %+v", decoded, layout)
|
||||
}
|
||||
if len(decoded.ExtentSizes) != len(layout.ExtentSizes) {
|
||||
t.Fatalf("extent count = %d, want %d", len(decoded.ExtentSizes), len(layout.ExtentSizes))
|
||||
}
|
||||
for i := range layout.ExtentSizes {
|
||||
if decoded.ExtentSizes[i] != layout.ExtentSizes[i] {
|
||||
t.Fatalf("extent %d = %d, want %d", i, decoded.ExtentSizes[i], layout.ExtentSizes[i])
|
||||
}
|
||||
}
|
||||
if string(decoded.Payload) != string(layout.Payload) {
|
||||
t.Fatalf("payload = %v, want %v", decoded.Payload, layout.Payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeLayoutRejectsCorruptInput(t *testing.T) {
|
||||
layout := &Layout{Format: "parquet", ExtentSizes: []int64{10, 20}, Align: 1}
|
||||
encoded, err := layout.Encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Encode() error = %v", err)
|
||||
}
|
||||
for cut := 0; cut < len(encoded); cut++ {
|
||||
if _, err := DecodeLayout(encoded[:cut]); err == nil {
|
||||
t.Fatalf("DecodeLayout() accepted truncation at %d", cut)
|
||||
}
|
||||
}
|
||||
if _, err := DecodeLayout(append(append([]byte{}, encoded...), 0)); err == nil {
|
||||
t.Fatalf("DecodeLayout() accepted trailing bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLayoutValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
layout Layout
|
||||
fileSize int64
|
||||
wantErr string
|
||||
}{
|
||||
{"valid", Layout{Format: "x", ExtentSizes: []int64{5, 5}, Align: 1}, 10, ""},
|
||||
{"skip size check", Layout{Format: "x", ExtentSizes: []int64{5}, Align: 1}, -1, ""},
|
||||
{"wrong total", Layout{Format: "x", ExtentSizes: []int64{5, 5}, Align: 1}, 11, "but the file has"},
|
||||
{"zero extent", Layout{Format: "x", ExtentSizes: []int64{5, 0}, Align: 1}, -1, "invalid size"},
|
||||
{"no extents", Layout{Format: "x", Align: 1}, -1, "no extents"},
|
||||
{"bad align", Layout{Format: "x", ExtentSizes: []int64{5}, Align: 0}, -1, "align"},
|
||||
{"no name", Layout{ExtentSizes: []int64{5}, Align: 1}, -1, "format name"},
|
||||
{"name too long", Layout{Format: strings.Repeat("x", MaxFormatNameBytes+1), ExtentSizes: []int64{5}, Align: 1}, -1, "too long"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
err := test.layout.Validate(test.fileSize)
|
||||
if test.wantErr == "" {
|
||||
if err != nil {
|
||||
t.Fatalf("%s: Validate() error = %v", test.name, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), test.wantErr) {
|
||||
t.Fatalf("%s: Validate() error = %v, want %q", test.name, err, test.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtentRange(t *testing.T) {
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{10, 20, 30}, Align: 1}
|
||||
offset, size, ok := layout.ExtentRange(1)
|
||||
if !ok || offset != 10 || size != 20 {
|
||||
t.Fatalf("ExtentRange(1) = (%d, %d, %v), want (10, 20, true)", offset, size, ok)
|
||||
}
|
||||
if _, _, ok := layout.ExtentRange(3); ok {
|
||||
t.Fatalf("ExtentRange(3) accepted out-of-range index")
|
||||
}
|
||||
if _, _, ok := layout.ExtentRange(-1); ok {
|
||||
t.Fatalf("ExtentRange(-1) accepted negative index")
|
||||
}
|
||||
}
|
||||
|
||||
// collectChunks walks the cutter the way the upload loop does.
|
||||
func collectChunks(t *testing.T, cutter *Cutter) [][2]int64 {
|
||||
t.Helper()
|
||||
var chunks [][2]int64
|
||||
var offset int64
|
||||
for {
|
||||
size := cutter.NextChunkSize(offset)
|
||||
if size <= 0 {
|
||||
return chunks
|
||||
}
|
||||
chunks = append(chunks, [2]int64{offset, size})
|
||||
offset += size
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutterKeepsExtentBoundaries(t *testing.T) {
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{5, 4}, Align: 1}
|
||||
chunks := collectChunks(t, layout.Cutter(16))
|
||||
want := [][2]int64{{0, 5}, {5, 4}}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("chunks = %v, want %v", chunks, want)
|
||||
}
|
||||
for i := range want {
|
||||
if chunks[i] != want[i] {
|
||||
t.Fatalf("chunk %d = %v, want %v", i, chunks[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutterSplitsOversizedExtentsOnAlign(t *testing.T) {
|
||||
// maxChunkSize 5 with align 2 quantizes down to 4-byte interior cuts.
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{10, 3}, Align: 2}
|
||||
chunks := collectChunks(t, layout.Cutter(5))
|
||||
want := [][2]int64{{0, 4}, {4, 4}, {8, 2}, {10, 3}}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("chunks = %v, want %v", chunks, want)
|
||||
}
|
||||
for i := range want {
|
||||
if chunks[i] != want[i] {
|
||||
t.Fatalf("chunk %d = %v, want %v", i, chunks[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutterAlignLargerThanChunkLimit(t *testing.T) {
|
||||
// Align above maxChunkSize still cuts on whole atoms.
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{20}, Align: 8}
|
||||
chunks := collectChunks(t, layout.Cutter(5))
|
||||
want := [][2]int64{{0, 8}, {8, 8}, {16, 4}}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("chunks = %v, want %v", chunks, want)
|
||||
}
|
||||
for i := range want {
|
||||
if chunks[i] != want[i] {
|
||||
t.Fatalf("chunk %d = %v, want %v", i, chunks[i], want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCutterUnlimitedKeepsOneChunkPerExtent(t *testing.T) {
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{10, 3}, Align: 188}
|
||||
chunks := collectChunks(t, layout.Cutter(0))
|
||||
want := [][2]int64{{0, 10}, {10, 3}}
|
||||
if len(chunks) != len(want) {
|
||||
t.Fatalf("chunks = %v, want %v", chunks, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A hostile layout may declare an enormous extent; the cutter must stay O(1)
|
||||
// per query instead of materializing every interior cut.
|
||||
func TestCutterHugeExtentStaysLazy(t *testing.T) {
|
||||
const quantum = 4 << 20 // 4MiB, already a multiple of align 1
|
||||
layout := &Layout{Format: "x", ExtentSizes: []int64{1 << 50, 188}, Align: 188}
|
||||
cutter := layout.Cutter(quantum)
|
||||
alignedQuantum := int64(quantum - quantum%188)
|
||||
if got := cutter.NextChunkSize(0); got != alignedQuantum {
|
||||
t.Fatalf("NextChunkSize(0) = %d, want %d", got, alignedQuantum)
|
||||
}
|
||||
if got := cutter.NextChunkSize(alignedQuantum * 1000); got != alignedQuantum {
|
||||
t.Fatalf("mid-extent chunk = %d, want %d", got, alignedQuantum)
|
||||
}
|
||||
// the final interior chunk stops at the extent boundary
|
||||
last := (int64(1<<50) / alignedQuantum) * alignedQuantum
|
||||
if got := cutter.NextChunkSize(last); got != int64(1<<50)-last {
|
||||
t.Fatalf("tail chunk = %d, want %d", got, int64(1<<50)-last)
|
||||
}
|
||||
// the next extent still cuts independently
|
||||
if got := cutter.NextChunkSize(1 << 50); got != 188 {
|
||||
t.Fatalf("second extent chunk = %d, want 188", got)
|
||||
}
|
||||
if got := cutter.NextChunkSize(1<<50 + 188); got != 0 {
|
||||
t.Fatalf("past end = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
// Package parquet adapts parquet files: extents are cut at row-group starts,
|
||||
// with one trailing extent for the page indexes and footer. Readers that fetch
|
||||
// row groups by offset then hit exactly the covering chunks. No view is
|
||||
// needed; the whole benefit is delivered by alignment.
|
||||
package parquet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
|
||||
parquetgo "github.com/parquet-go/parquet-go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
)
|
||||
|
||||
const FormatName = "parquet"
|
||||
|
||||
var magic = []byte("PAR1")
|
||||
|
||||
func init() {
|
||||
format.Register(Adapter{})
|
||||
}
|
||||
|
||||
type Adapter struct{}
|
||||
|
||||
var (
|
||||
_ format.Sniffer = Adapter{}
|
||||
_ format.Indexer = Adapter{}
|
||||
)
|
||||
|
||||
func (Adapter) Name() string { return FormatName }
|
||||
|
||||
func (Adapter) Sniff(h format.Hint) bool {
|
||||
return bytes.HasPrefix(h.Head, magic) && bytes.HasSuffix(h.Tail, magic)
|
||||
}
|
||||
|
||||
// Index reads the footer and cuts one extent per row group. The leading magic
|
||||
// rides with the first row group; everything after the last row group (page
|
||||
// indexes, footer) forms the final extent.
|
||||
func (Adapter) Index(ctx context.Context, r io.ReaderAt, size int64) (*format.Layout, error) {
|
||||
file, err := parquetgo.OpenFile(r, size, parquetgo.SkipPageIndex(true), parquetgo.SkipBloomFilters(true))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open parquet: %w", err)
|
||||
}
|
||||
rowGroups := file.Metadata().RowGroups
|
||||
if len(rowGroups) == 0 {
|
||||
return nil, fmt.Errorf("parquet file has no row groups")
|
||||
}
|
||||
if len(rowGroups) >= format.MaxExtentCount {
|
||||
return nil, fmt.Errorf("parquet file has too many row groups: %d", len(rowGroups))
|
||||
}
|
||||
|
||||
starts := make([]int64, 0, len(rowGroups))
|
||||
var lastEnd int64
|
||||
for i, rowGroup := range rowGroups {
|
||||
start, end := int64(math.MaxInt64), int64(0)
|
||||
for _, column := range rowGroup.Columns {
|
||||
columnStart := column.MetaData.DataPageOffset
|
||||
if dictionary := column.MetaData.DictionaryPageOffset; dictionary > 0 && dictionary < columnStart {
|
||||
columnStart = dictionary
|
||||
}
|
||||
if columnStart < start {
|
||||
start = columnStart
|
||||
}
|
||||
if columnEnd := columnStart + column.MetaData.TotalCompressedSize; columnEnd > end {
|
||||
end = columnEnd
|
||||
}
|
||||
}
|
||||
if len(rowGroup.Columns) == 0 || start >= end || start < int64(len(magic)) || end > size {
|
||||
return nil, fmt.Errorf("row group %d has an invalid byte range [%d, %d)", i, start, end)
|
||||
}
|
||||
if i > 0 && start < lastEnd {
|
||||
return nil, fmt.Errorf("row group %d overlaps its predecessor", i)
|
||||
}
|
||||
starts = append(starts, start)
|
||||
lastEnd = end
|
||||
}
|
||||
if lastEnd >= size {
|
||||
return nil, fmt.Errorf("row groups leave no room for the footer")
|
||||
}
|
||||
|
||||
var sizes []int64
|
||||
var previous int64
|
||||
for _, start := range starts[1:] {
|
||||
sizes = append(sizes, start-previous)
|
||||
previous = start
|
||||
}
|
||||
sizes = append(sizes, lastEnd-previous)
|
||||
sizes = append(sizes, size-lastEnd)
|
||||
|
||||
layout := &format.Layout{Format: FormatName, ExtentSizes: sizes, Align: 1}
|
||||
if err := layout.Validate(size); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return layout, nil
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package parquet
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
parquetgo "github.com/parquet-go/parquet-go"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format/formattest"
|
||||
)
|
||||
|
||||
type row struct {
|
||||
ID int64 `parquet:"id"`
|
||||
Name string `parquet:"name"`
|
||||
}
|
||||
|
||||
// buildParquet writes rowGroups row groups of rowsPerGroup rows each.
|
||||
func buildParquet(t *testing.T, rowGroups, rowsPerGroup int) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
writer := parquetgo.NewGenericWriter[row](&buf)
|
||||
for g := 0; g < rowGroups; g++ {
|
||||
rows := make([]row, rowsPerGroup)
|
||||
for i := range rows {
|
||||
rows[i] = row{ID: int64(g*rowsPerGroup + i), Name: "some filler content to give row groups a little size"}
|
||||
}
|
||||
if _, err := writer.Write(rows); err != nil {
|
||||
t.Fatalf("Write() error = %v", err)
|
||||
}
|
||||
if err := writer.Flush(); err != nil {
|
||||
t.Fatalf("Flush() error = %v", err)
|
||||
}
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
t.Fatalf("Close() error = %v", err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func TestIndex(t *testing.T) {
|
||||
data := buildParquet(t, 3, 100)
|
||||
size := int64(len(data))
|
||||
layout, err := Adapter{}.Index(context.Background(), bytes.NewReader(data), size)
|
||||
if err != nil {
|
||||
t.Fatalf("Index() error = %v", err)
|
||||
}
|
||||
// one extent per row group plus the trailing footer extent
|
||||
if len(layout.ExtentSizes) != 4 {
|
||||
t.Fatalf("extents = %v, want 4", layout.ExtentSizes)
|
||||
}
|
||||
if err := layout.Validate(size); err != nil {
|
||||
t.Fatalf("Validate() error = %v", err)
|
||||
}
|
||||
formattest.EncodeRoundTrip(t, layout)
|
||||
|
||||
// extent cuts must land on the row-group starts the footer declares
|
||||
file, err := parquetgo.OpenFile(bytes.NewReader(data), size)
|
||||
if err != nil {
|
||||
t.Fatalf("OpenFile() error = %v", err)
|
||||
}
|
||||
var offset int64
|
||||
for i, extentSize := range layout.ExtentSizes[:3] {
|
||||
if i > 0 {
|
||||
want := file.Metadata().RowGroups[i].Columns[0].MetaData.DataPageOffset
|
||||
if dictionary := file.Metadata().RowGroups[i].Columns[0].MetaData.DictionaryPageOffset; dictionary > 0 && dictionary < want {
|
||||
want = dictionary
|
||||
}
|
||||
if offset != want {
|
||||
t.Fatalf("extent %d starts at %d, row group starts at %d", i, offset, want)
|
||||
}
|
||||
}
|
||||
offset += extentSize
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexSingleRowGroup(t *testing.T) {
|
||||
data := buildParquet(t, 1, 10)
|
||||
layout, err := Adapter{}.Index(context.Background(), bytes.NewReader(data), int64(len(data)))
|
||||
if err != nil {
|
||||
t.Fatalf("Index() error = %v", err)
|
||||
}
|
||||
if len(layout.ExtentSizes) != 2 {
|
||||
t.Fatalf("extents = %v, want 2", layout.ExtentSizes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexRejectsNonParquet(t *testing.T) {
|
||||
data := []byte("this is not a parquet file, not even close, but long enough")
|
||||
if _, err := (Adapter{}).Index(context.Background(), bytes.NewReader(data), int64(len(data))); err == nil {
|
||||
t.Fatalf("Index() accepted junk")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexTruncations(t *testing.T) {
|
||||
formattest.IndexTruncations(t, Adapter{}, buildParquet(t, 2, 50))
|
||||
}
|
||||
|
||||
func TestSniff(t *testing.T) {
|
||||
data := buildParquet(t, 1, 10)
|
||||
head, tail := data[:4], data[len(data)-4:]
|
||||
if !(Adapter{}).Sniff(format.Hint{Head: head, Tail: tail}) {
|
||||
t.Fatalf("Sniff() rejected parquet magic")
|
||||
}
|
||||
if (Adapter{}).Sniff(format.Hint{Head: []byte("PAR1"), Tail: []byte("nope")}) {
|
||||
t.Fatalf("Sniff() accepted missing tail magic")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package format
|
||||
|
||||
// Adapters register from init(), so the map needs no locking.
|
||||
var formats = map[string]Format{}
|
||||
|
||||
func Register(f Format) {
|
||||
if _, ok := formats[f.Name()]; ok {
|
||||
panic("format: duplicate adapter " + f.Name())
|
||||
}
|
||||
formats[f.Name()] = f
|
||||
}
|
||||
|
||||
// ByName returns the registered adapter, or nil.
|
||||
func ByName(name string) Format {
|
||||
return formats[name]
|
||||
}
|
||||
@@ -139,7 +139,7 @@ func (s3a *S3ApiServer) PostPolicyBucketHandler(w http.ResponseWriter, r *http.R
|
||||
// Forward validated POST form fields to the underlying PUT as headers.
|
||||
applyPostPolicyFormHeaders(r, formValues)
|
||||
|
||||
// Authorize the object like the PUT path.
|
||||
// Authorize the object like the PUT path; the coarse Write check above does not consult the bucket policy.
|
||||
if errCode := s3a.iam.AuthorizeObjectWrite(r, identity, bucket, object); errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, refineDenialForBucketName(r, identity, errCode))
|
||||
return
|
||||
@@ -382,15 +382,25 @@ func getRedirectPostRawQuery(bucket, key, etag string) string {
|
||||
}
|
||||
|
||||
// Check to see if Policy is signed correctly, returning the signing identity.
|
||||
// The verified form's access key is recorded in ctx's identity holder.
|
||||
// Write authorization happens later in the handler via AuthorizeObjectWrite,
|
||||
// which evaluates the identity's attached IAM policies, session policies, and
|
||||
// the bucket policy against the real object key; gating here on the legacy
|
||||
// Actions projection would deny identities those mechanisms allow.
|
||||
// The verified form's access key is recorded in ctx's identity holder. The
|
||||
// identity's Write permission on the bucket is checked only once the signature
|
||||
// verifies, as on the header-signed path, so a form that fails verification
|
||||
// learns nothing about the key's permissions. A verified identity that may not
|
||||
// write is returned with ErrAccessDenied.
|
||||
func (iam *IdentityAccessManagement) doesPolicySignatureMatch(ctx context.Context, formValues http.Header) (*Identity, s3err.ErrorCode) {
|
||||
var identity *Identity
|
||||
var errCode s3err.ErrorCode
|
||||
// For SignV2 - Signature field will be valid
|
||||
if _, ok := formValues["Signature"]; ok {
|
||||
return iam.doesPolicySignatureV2Match(ctx, formValues)
|
||||
identity, errCode = iam.doesPolicySignatureV2Match(ctx, formValues)
|
||||
} else {
|
||||
identity, errCode = iam.doesPolicySignatureV4Match(ctx, formValues)
|
||||
}
|
||||
return iam.doesPolicySignatureV4Match(ctx, formValues)
|
||||
if errCode != s3err.ErrNone {
|
||||
return nil, errCode
|
||||
}
|
||||
if !identity.CanDo(s3_constants.ACTION_WRITE, formValues.Get("bucket"), "") {
|
||||
return identity, s3err.ErrAccessDenied
|
||||
}
|
||||
return identity, s3err.ErrNone
|
||||
}
|
||||
@@ -32,11 +32,6 @@ const (
|
||||
postAclAccessKey = "AKIAACLTESTTEST"
|
||||
postAclSecretKey = "secret-key-for-acl-tests"
|
||||
postAclPrincipal = "arn:aws:iam::000000000000:user/aclwriter"
|
||||
|
||||
postPolicyAccessKey = "AKIAPOLICYONLY0000"
|
||||
postPolicySecretKey = "secret-key-for-policy-only"
|
||||
postReadAccessKey = "AKIAPOLICYREAD0000"
|
||||
postReadSecretKey = "secret-key-for-policy-read"
|
||||
)
|
||||
|
||||
// newPostAuthzServer builds an S3ApiServer whose non-admin "tester" identity
|
||||
@@ -279,118 +274,3 @@ func TestAuthorizeObjectAclWrite(t *testing.T) {
|
||||
s3a.iam.AuthorizeObjectAclWrite(req, admin, postAuthzBucket, "acl-denied/secret.txt"),
|
||||
"an admin is not confined by the bucket policy")
|
||||
}
|
||||
|
||||
// newPostPolicyOnlyServer builds an S3ApiServer with identities that hold no
|
||||
// legacy Write action: "poster" carries an attached IAM policy allowing
|
||||
// s3:PutObject on the test bucket, and "reader" one allowing only s3:GetObject.
|
||||
func newPostPolicyOnlyServer(t *testing.T) *S3ApiServer {
|
||||
t.Helper()
|
||||
|
||||
iam := &IdentityAccessManagement{
|
||||
hashes: make(map[string]*sync.Pool),
|
||||
hashCounters: make(map[string]*int32),
|
||||
}
|
||||
err := iam.loadS3ApiConfiguration(&iam_pb.S3ApiConfiguration{
|
||||
Identities: []*iam_pb.Identity{{
|
||||
Name: "poster",
|
||||
Account: &iam_pb.Account{Id: "000000000000", DisplayName: "poster"},
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: postPolicyAccessKey, SecretKey: postPolicySecretKey}},
|
||||
Actions: []string{"List:" + postAuthzBucket},
|
||||
PolicyNames: []string{"post-write"},
|
||||
}, {
|
||||
Name: "reader",
|
||||
Account: &iam_pb.Account{Id: "000000000000", DisplayName: "reader"},
|
||||
Credentials: []*iam_pb.Credential{{AccessKey: postReadAccessKey, SecretKey: postReadSecretKey}},
|
||||
Actions: []string{"Read:" + postAuthzBucket, "List:" + postAuthzBucket},
|
||||
PolicyNames: []string{"post-read"},
|
||||
}},
|
||||
Policies: []*iam_pb.Policy{{
|
||||
Name: "post-write",
|
||||
Content: fmt.Sprintf(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject", "s3:GetObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::%s/*"]
|
||||
}, {
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::%s"]
|
||||
}]
|
||||
}`, postAuthzBucket, postAuthzBucket),
|
||||
}, {
|
||||
Name: "post-read",
|
||||
Content: fmt.Sprintf(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::%s/*"]
|
||||
}]
|
||||
}`, postAuthzBucket),
|
||||
}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
s3a := &S3ApiServer{
|
||||
option: &S3ApiServerOption{BucketsPath: "/buckets"},
|
||||
iam: iam,
|
||||
}
|
||||
s3a.bucketRegistry = NewBucketRegistry(s3a)
|
||||
s3a.bucketRegistry.setMetadataCache(&BucketMetaData{Name: postAuthzBucket, IsTableBucket: false})
|
||||
return s3a
|
||||
}
|
||||
|
||||
// postFormValues canonicalizes the signed request's multipart fields the way
|
||||
// extractPostPolicyFormValues does, for driving doesPolicySignatureMatch.
|
||||
func postFormValues(t *testing.T, r *http.Request) http.Header {
|
||||
t.Helper()
|
||||
require.NoError(t, r.ParseMultipartForm(1<<20))
|
||||
formValues := make(http.Header)
|
||||
for k, v := range r.MultipartForm.Value {
|
||||
formValues[http.CanonicalHeaderKey(k)] = v
|
||||
}
|
||||
return formValues
|
||||
}
|
||||
|
||||
// TestPostPolicySignatureAllowsAttachedPolicyIdentity signs a POST Object form
|
||||
// with a key whose holder has no legacy Write action - only an attached IAM
|
||||
// policy granting s3:PutObject - and asserts the signature check accepts it
|
||||
// and the shared object-write authorization allows the key. Gating the
|
||||
// signature check on the Actions projection used to fail this with
|
||||
// AccessDenied before authorization ever ran.
|
||||
func TestPostPolicySignatureAllowsAttachedPolicyIdentity(t *testing.T) {
|
||||
s3a := newPostPolicyOnlyServer(t)
|
||||
|
||||
req := newSignedPostRequestFor(t, "uploads/pic.png", postPolicyAccessKey, postPolicySecretKey, nil)
|
||||
identity, errCode := s3a.iam.doesPolicySignatureMatch(req.Context(), postFormValues(t, req))
|
||||
require.Equal(t, s3err.ErrNone, errCode,
|
||||
"a verified POST signature must not require the legacy Write action")
|
||||
require.NotNil(t, identity)
|
||||
|
||||
assert.Equal(t, s3err.ErrNone,
|
||||
s3a.iam.AuthorizeObjectWrite(req, identity, postAuthzBucket, "uploads/pic.png"),
|
||||
"an attached IAM policy granting s3:PutObject must authorize the key like the PUT path")
|
||||
}
|
||||
|
||||
// TestPostPolicyAttachedPolicyDeny covers the deny leg: a verified identity
|
||||
// whose attached policy does not grant s3:PutObject is still refused by
|
||||
// AuthorizeObjectWrite and by the full handler.
|
||||
func TestPostPolicyAttachedPolicyDeny(t *testing.T) {
|
||||
s3a := newPostPolicyOnlyServer(t)
|
||||
|
||||
req := newSignedPostRequestFor(t, "uploads/pic.png", postReadAccessKey, postReadSecretKey, nil)
|
||||
identity, errCode := s3a.iam.doesPolicySignatureMatch(req.Context(), postFormValues(t, req))
|
||||
require.Equal(t, s3err.ErrNone, errCode)
|
||||
require.NotNil(t, identity)
|
||||
|
||||
assert.Equal(t, s3err.ErrAccessDenied,
|
||||
s3a.iam.AuthorizeObjectWrite(req, identity, postAuthzBucket, "uploads/pic.png"),
|
||||
"an attached policy without s3:PutObject must still deny the upload")
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
s3a.PostPolicyBucketHandler(rec, newSignedPostRequestFor(t, "uploads/pic.png", postReadAccessKey, postReadSecretKey, nil))
|
||||
assert.Equal(t, http.StatusForbidden, rec.Code,
|
||||
"POST from a read-only identity must stay AccessDenied, body: %s", rec.Body.String())
|
||||
assert.Contains(t, rec.Body.String(), "AccessDenied")
|
||||
}
|
||||
@@ -0,0 +1,621 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/format/hlsts"
|
||||
_ "github.com/seaweedfs/seaweedfs/weed/format/parquet"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/constants"
|
||||
)
|
||||
|
||||
const (
|
||||
// Query parameters follow the mv.from/cp.from dotted convention so the
|
||||
// general POST endpoint cannot collide with pass-through client params.
|
||||
formatIngestParam = "format.ingest"
|
||||
formatRepackParam = "format.repack"
|
||||
|
||||
// formatLayoutChunksKey binds a layout to the chunk list it described, so
|
||||
// any other writer that changes the chunks invalidates the views.
|
||||
formatLayoutChunksKey = "x-seaweedfs-format-layout-chunks"
|
||||
|
||||
maxFormatSidecarBytes = 16 << 20
|
||||
formatSniffBytes = 512
|
||||
// defaultFormatChunkSizeMB caps extent chunks when no maxMB is configured.
|
||||
// Extent chunks are buffered in memory, so the limit must never be absent.
|
||||
defaultFormatChunkSizeMB = 4
|
||||
)
|
||||
|
||||
// formatChunkIdentity digests the chunk list a layout was written against,
|
||||
// covering every field that changes what a read returns: a FUSE truncate, for
|
||||
// one, mutates Size while keeping the chunk's id.
|
||||
func formatChunkIdentity(chunks []*filer_pb.FileChunk) []byte {
|
||||
digest := md5.New()
|
||||
for _, chunk := range chunks {
|
||||
fmt.Fprintf(digest, "%d:%s:%d:%d:%x:%t:%t:%d;",
|
||||
chunk.Offset, chunk.GetFileIdString(), chunk.Size, chunk.ModifiedTsNs,
|
||||
chunk.CipherKey, chunk.IsCompressed, chunk.IsChunkManifest, chunk.SseType)
|
||||
}
|
||||
return digest.Sum(nil)
|
||||
}
|
||||
|
||||
// repackSourceIdentity digests every entry field that influenced a repack's
|
||||
// output: the chunk list it read, the size the layout was validated against,
|
||||
// the TTL and expiry anchors its new chunks were assigned with, and the
|
||||
// hard-link and remote state its guards evaluated.
|
||||
func repackSourceIdentity(entry *filer.Entry) []byte {
|
||||
digest := md5.New()
|
||||
digest.Write(formatChunkIdentity(entry.GetChunks()))
|
||||
digest.Write(entry.Content)
|
||||
fmt.Fprintf(digest, "%d:%d:%d:%d:%d:%t:%x:%t",
|
||||
len(entry.Content), entry.FileSize, entry.TtlSec, entry.Crtime.UnixNano(), entry.Mtime.UnixNano(),
|
||||
entry.IsExpireS3Enabled(), []byte(entry.HardLinkId), entry.Remote != nil)
|
||||
return digest.Sum(nil)
|
||||
}
|
||||
|
||||
// roundUpToVolumeTTL returns the smallest volume-TTL-representable seconds
|
||||
// value not below the argument. A volume TTL is at most 255 of one unit and
|
||||
// SecondsToTTL truncates anything else downward, which would let chunks
|
||||
// expire before their entry - or, under a minute, never.
|
||||
func roundUpToVolumeTTL(seconds int64) int32 {
|
||||
for _, unit := range []int64{60, 3600, 24 * 3600, 7 * 24 * 3600, 30 * 24 * 3600, 365 * 24 * 3600} {
|
||||
count := (seconds + unit - 1) / unit
|
||||
if count <= 255 && count*unit <= math.MaxInt32 {
|
||||
return int32(count * unit)
|
||||
}
|
||||
}
|
||||
// Nothing above ~68 years rounds up within int32; no volume TTL keeps the
|
||||
// chunks past the entry, which is the safe direction.
|
||||
return 0
|
||||
}
|
||||
|
||||
// formatChunkSizeLimit mirrors the autoChunk maxMB resolution.
|
||||
func (fs *FilerServer) formatChunkSizeLimit(r *http.Request) int64 {
|
||||
parsedMaxMB, _ := strconv.ParseInt(r.URL.Query().Get("maxMB"), 10, 32)
|
||||
maxMB := int32(parsedMaxMB)
|
||||
if maxMB <= 0 && fs.option.MaxMB > 0 {
|
||||
maxMB = int32(fs.option.MaxMB)
|
||||
}
|
||||
if maxMB <= 0 {
|
||||
maxMB = defaultFormatChunkSizeMB
|
||||
}
|
||||
return int64(maxMB) * 1024 * 1024
|
||||
}
|
||||
|
||||
// copyStandardHeadersToExtended matches what saveMetaData keeps on an entry.
|
||||
func copyStandardHeadersToExtended(r *http.Request, extended map[string][]byte) {
|
||||
for k, v := range r.Header {
|
||||
if len(v) > 0 && len(v[0]) > 0 {
|
||||
if strings.HasPrefix(k, needle.PairNamePrefix) || k == "Cache-Control" || k == "Expires" || k == "Content-Disposition" {
|
||||
extended[k] = []byte(v[0])
|
||||
}
|
||||
if k == "Response-Content-Disposition" {
|
||||
extended["Content-Disposition"] = []byte(v[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// formatIngest handles POST /path?format.ingest=<adapter>: a multipart body
|
||||
// with an "index" sidecar part describing the media's extents, then the
|
||||
// "media" bytes. Storage chunks are cut on the boundaries the sidecar declares.
|
||||
func (fs *FilerServer) formatIngest(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
|
||||
adapterName := r.URL.Query().Get(formatIngestParam)
|
||||
adapter := format.ByName(adapterName)
|
||||
if adapter == nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName))
|
||||
return
|
||||
}
|
||||
sidecarIndexer, ok := adapter.(format.SidecarIndexer)
|
||||
if !ok {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("format %q does not support sidecar ingest", adapterName))
|
||||
return
|
||||
}
|
||||
if strings.HasSuffix(r.URL.Path, "/") {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("format ingest target must be a file path"))
|
||||
return
|
||||
}
|
||||
if enforced, err := fs.wormEnforcedForEntry(ctx, r.URL.Path); err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
} else if enforced {
|
||||
writeJsonError(w, r, http.StatusForbidden, errors.New("cannot replace WORM-enforced entry"))
|
||||
return
|
||||
}
|
||||
|
||||
multipartReader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("format ingest requires multipart/form-data: %w", err))
|
||||
return
|
||||
}
|
||||
sidecarPart, err := multipartReader.NextPart()
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("read index part: %w", err))
|
||||
return
|
||||
}
|
||||
if sidecarPart.FormName() != "index" {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("first multipart part must be named index"))
|
||||
return
|
||||
}
|
||||
sidecar, err := io.ReadAll(io.LimitReader(sidecarPart, maxFormatSidecarBytes+1))
|
||||
sidecarPart.Close()
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("read index part: %w", err))
|
||||
return
|
||||
}
|
||||
if len(sidecar) > maxFormatSidecarBytes {
|
||||
writeJsonError(w, r, http.StatusRequestEntityTooLarge, fmt.Errorf("index part exceeds %d bytes", maxFormatSidecarBytes))
|
||||
return
|
||||
}
|
||||
layout, err := sidecarIndexer.IndexSidecar(sidecar)
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if err := layout.Validate(-1); err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
mediaPart, err := multipartReader.NextPart()
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("read media part: %w", err))
|
||||
return
|
||||
}
|
||||
if mediaPart.FormName() != "media" {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("second multipart part must be named media"))
|
||||
return
|
||||
}
|
||||
defer mediaPart.Close()
|
||||
contentType := mediaPart.Header.Get("Content-Type")
|
||||
if contentType == "" {
|
||||
contentType = "application/octet-stream"
|
||||
}
|
||||
|
||||
cutter := layout.Cutter(fs.formatChunkSizeLimit(r))
|
||||
fileChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, mediaPart, 0, cutter, path.Base(r.URL.Path), contentType, false, so)
|
||||
cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), fileChunks) }
|
||||
if uploadErr != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, uploadErr)
|
||||
return
|
||||
}
|
||||
expectedMd5 := mediaPart.Header.Get("Content-Md5")
|
||||
if expectedMd5 == "" {
|
||||
expectedMd5 = r.Header.Get("Content-Md5")
|
||||
}
|
||||
if expectedMd5 != "" {
|
||||
if md5bytes := md5Hash.Sum(nil); util.Base64Encode(md5bytes) != expectedMd5 && fmt.Sprintf("%x", md5bytes) != expectedMd5 {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New(constants.ErrMsgBadDigest))
|
||||
return
|
||||
}
|
||||
}
|
||||
if total := layout.TotalSize(); written != total {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("media is %d bytes but the index describes %d", written, total))
|
||||
return
|
||||
}
|
||||
var extra [1]byte
|
||||
if n, _ := io.ReadFull(mediaPart, extra[:]); n != 0 {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("media has trailing bytes beyond the index"))
|
||||
return
|
||||
}
|
||||
if extraPart, nextErr := multipartReader.NextPart(); nextErr == nil {
|
||||
extraPart.Close()
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("unexpected multipart part after media"))
|
||||
return
|
||||
}
|
||||
|
||||
fileChunks, err = filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, fileChunks)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
encoded, err := layout.Encode()
|
||||
if err != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
mode := uint64(0660)
|
||||
if text := r.URL.Query().Get("mode"); text != "" {
|
||||
if mode, err = strconv.ParseUint(text, 8, 32); err != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("invalid mode %q", text))
|
||||
return
|
||||
}
|
||||
}
|
||||
now := time.Now()
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.FullPath(r.URL.Path),
|
||||
Attr: filer.Attr{
|
||||
Mtime: now, Crtime: now,
|
||||
Mode: os.FileMode(mode), Uid: OS_UID, Gid: OS_GID,
|
||||
TtlSec: so.TtlSeconds, Mime: contentType,
|
||||
Md5: md5Hash.Sum(nil), FileSize: uint64(written),
|
||||
},
|
||||
Chunks: fileChunks,
|
||||
Extended: map[string][]byte{
|
||||
format.LayoutKey: encoded,
|
||||
formatLayoutChunksKey: formatChunkIdentity(fileChunks),
|
||||
},
|
||||
}
|
||||
copyStandardHeadersToExtended(r, entry.Extended)
|
||||
// commit under the entry lock like saveMetaData, so ingest overwrites
|
||||
// serialize with gRPC writers, renames, and repack
|
||||
pathLock := fs.entryLockTable.AcquireLock("formatIngest", entry.FullPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(entry.FullPath, pathLock)
|
||||
// recheck under the lock: WORM may have been enabled during the upload
|
||||
if enforced, wormErr := fs.wormEnforcedForEntry(ctx, r.URL.Path); wormErr != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, wormErr)
|
||||
return
|
||||
} else if enforced {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusForbidden, errors.New("cannot replace WORM-enforced entry"))
|
||||
return
|
||||
}
|
||||
if err := fs.filer.CreateEntry(context.WithoutCancel(ctx), entry, nil, false, false, nil, skipCheckParentDirEntry(r), so.MaxFileNameLength); err != nil {
|
||||
// CreateEntry can persist the entry and then fail creating parents;
|
||||
// keep chunks the stored entry still references
|
||||
fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), fs.inlineAppendChunksNotReferenced(ctx, entry.FullPath, fileChunks))
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
writeJsonQuiet(w, r, http.StatusCreated, FilerPostResult{Name: entry.Name(), Size: written})
|
||||
}
|
||||
|
||||
// formatRepack handles POST /path?format.repack=<adapter>: it derives the
|
||||
// layout from the stored bytes and rewrites the entry's chunks cut on extent
|
||||
// boundaries. The bytes do not change, only where they are cut.
|
||||
func (fs *FilerServer) formatRepack(ctx context.Context, w http.ResponseWriter, r *http.Request, so *operation.StorageOption) {
|
||||
adapterName := r.URL.Query().Get(formatRepackParam)
|
||||
adapter := format.ByName(adapterName)
|
||||
if adapter == nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("unknown format %q", adapterName))
|
||||
return
|
||||
}
|
||||
indexer, ok := adapter.(format.Indexer)
|
||||
if !ok {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("format %q does not support repack", adapterName))
|
||||
return
|
||||
}
|
||||
fullPath := util.FullPath(r.URL.Path)
|
||||
// Serializes gRPC writers, renames, and this filer's HTTP overwrites;
|
||||
// cross-filer serialization needs owner routing.
|
||||
pathLock := fs.entryLockTable.AcquireLock("formatRepack", fullPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(fullPath, pathLock)
|
||||
|
||||
// checked under the lock so a concurrent WORM enable cannot land between
|
||||
// validation and the entry swap
|
||||
if enforced, err := fs.wormEnforcedForEntry(ctx, r.URL.Path); err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
} else if enforced {
|
||||
writeJsonError(w, r, http.StatusForbidden, errors.New("cannot repack WORM-enforced entry"))
|
||||
return
|
||||
}
|
||||
|
||||
entry, err := fs.filer.FindEntry(ctx, fullPath)
|
||||
if err != nil {
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
writeJsonError(w, r, http.StatusNotFound, err)
|
||||
} else {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if entry.IsDirectory() {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("cannot repack a directory"))
|
||||
return
|
||||
}
|
||||
oldChunks := entry.GetChunks()
|
||||
sourceIdentity := repackSourceIdentity(entry)
|
||||
if len(oldChunks) == 0 {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("entry has no chunks to repack"))
|
||||
return
|
||||
}
|
||||
if len(entry.HardLinkId) != 0 || entry.Remote != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("cannot repack hard-linked or remote entries"))
|
||||
return
|
||||
}
|
||||
// the repack reader sees only chunks; inline content would be dropped
|
||||
if len(entry.Content) != 0 {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("cannot repack entries with inline content"))
|
||||
return
|
||||
}
|
||||
for _, chunk := range oldChunks {
|
||||
if chunk.SseType != filer_pb.SSEType_NONE {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("cannot repack server-side encrypted entries"))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Repack rewrites where bytes are cut, never their lifetime: new chunks
|
||||
// carry the entry's remaining TTL, not the request query's nor a restart
|
||||
// of the original span.
|
||||
so.TtlSeconds = entry.TtlSec
|
||||
if entry.TtlSec > 0 {
|
||||
// mirror FindEntry's expiry anchors: S3-expiring entries age from
|
||||
// Mtime, everything else from Crtime
|
||||
expiresAt := entry.Crtime.Add(time.Duration(entry.TtlSec) * time.Second)
|
||||
if entry.IsExpireS3Enabled() {
|
||||
expiresAt = entry.GetS3ExpireTime()
|
||||
}
|
||||
remaining := (int64(time.Until(expiresAt)) + int64(time.Second) - 1) / int64(time.Second)
|
||||
if remaining <= 0 {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New("entry TTL has already expired"))
|
||||
return
|
||||
}
|
||||
so.TtlSeconds = roundUpToVolumeTTL(remaining)
|
||||
}
|
||||
|
||||
if entry.FileSize > math.MaxInt64 {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("file size %d overflows int64", entry.FileSize))
|
||||
return
|
||||
}
|
||||
size := int64(entry.FileSize)
|
||||
lookup := fs.filer.MasterClient.GetLookupFileIdFunction()
|
||||
chunkViews, err := filer.ViewFromChunks(ctx, lookup, oldChunks, 0, size)
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
readerCache := filer.NewReaderCache(8, chunk_cache.NewChunkCacheInMemory(16), lookup, nil)
|
||||
readerAt := filer.NewChunkReaderAtFromClient(ctx, readerCache, chunkViews, size, filer.DefaultPrefetchCount)
|
||||
// Close releases the private reader cache and its in-flight prefetches.
|
||||
defer readerAt.Close()
|
||||
|
||||
// cheap magic-byte gate before handing the bytes to the parser
|
||||
if sniffer, ok := adapter.(format.Sniffer); ok {
|
||||
hint := format.Hint{Name: entry.Name(), ContentType: entry.Attr.Mime, Size: size}
|
||||
sniffSize := int64(formatSniffBytes)
|
||||
if sniffSize > size {
|
||||
sniffSize = size
|
||||
}
|
||||
head := make([]byte, sniffSize)
|
||||
if _, err := readerAt.ReadAt(head, 0); err != nil && err != io.EOF {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("read head: %w", err))
|
||||
return
|
||||
}
|
||||
tail := make([]byte, sniffSize)
|
||||
if _, err := readerAt.ReadAt(tail, size-sniffSize); err != nil && err != io.EOF {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("read tail: %w", err))
|
||||
return
|
||||
}
|
||||
hint.Head, hint.Tail = head, tail
|
||||
if !sniffer.Sniff(hint) {
|
||||
writeJsonError(w, r, http.StatusBadRequest, fmt.Errorf("%s does not look like %s", entry.Name(), adapterName))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
layout, err := indexer.Index(ctx, readerAt, size)
|
||||
if err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
if err := layout.Validate(size); err != nil {
|
||||
writeJsonError(w, r, http.StatusBadRequest, err)
|
||||
return
|
||||
}
|
||||
|
||||
cutter := layout.Cutter(fs.formatChunkSizeLimit(r))
|
||||
newChunks, md5Hash, written, uploadErr, _ := fs.uploadReaderToBoundedChunks(ctx, r, io.NewSectionReader(readerAt, 0, size), 0, cutter, entry.Name(), entry.Attr.Mime, false, so)
|
||||
cleanup := func() { fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), newChunks) }
|
||||
if uploadErr != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, uploadErr)
|
||||
return
|
||||
}
|
||||
if written != size {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("read %d of %d bytes", written, size))
|
||||
return
|
||||
}
|
||||
newChunks, err = filer.MaybeManifestize(fs.saveAsChunk(ctx, so), fs.filer.DeleteChunksNotRecursive, newChunks)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
encoded, err := layout.Encode()
|
||||
if err != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
|
||||
// The entry lock is filer-local, so a writer on another filer is not
|
||||
// blocked by it. Re-read from the store and conflict on any change to
|
||||
// state that influenced this repack - chunks, size, TTL and expiry
|
||||
// anchors, hard-link and remote state - so the swap can never pair fresh
|
||||
// metadata with chunks built from stale inputs. The unguarded window
|
||||
// shrinks from the whole repack to this commit; within it repack races
|
||||
// like any ordinary writer. Closing it needs owner routing.
|
||||
current, err := fs.filer.FindEntry(ctx, fullPath)
|
||||
if err != nil {
|
||||
cleanup()
|
||||
if errors.Is(err, filer_pb.ErrNotFound) {
|
||||
writeJsonError(w, r, http.StatusConflict, errors.New("entry was deleted during repack"))
|
||||
} else {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !bytes.Equal(repackSourceIdentity(current), sourceIdentity) {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusConflict, errors.New("entry changed during repack"))
|
||||
return
|
||||
}
|
||||
if enforced, wormErr := fs.wormEnforcedForEntry(ctx, r.URL.Path); wormErr != nil {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusInternalServerError, wormErr)
|
||||
return
|
||||
} else if enforced {
|
||||
cleanup()
|
||||
writeJsonError(w, r, http.StatusForbidden, errors.New("cannot repack WORM-enforced entry"))
|
||||
return
|
||||
}
|
||||
|
||||
// build from the fresh read so a concurrent metadata-only update on
|
||||
// another filer is carried forward, not clobbered
|
||||
newEntry := *current
|
||||
newEntry.Chunks = newChunks
|
||||
newEntry.Extended = make(map[string][]byte)
|
||||
for k, v := range current.Extended {
|
||||
newEntry.Extended[k] = v
|
||||
}
|
||||
newEntry.Extended[format.LayoutKey] = encoded
|
||||
newEntry.Extended[formatLayoutChunksKey] = formatChunkIdentity(newChunks)
|
||||
if len(newEntry.Md5) == 0 {
|
||||
newEntry.Md5 = md5Hash.Sum(nil)
|
||||
}
|
||||
if err := fs.filer.UpdateEntry(context.WithoutCancel(ctx), current, &newEntry, false); err != nil {
|
||||
fs.filer.DeleteUncommittedChunks(context.WithoutCancel(ctx), fs.inlineAppendChunksNotReferenced(ctx, fullPath, newChunks))
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
return
|
||||
}
|
||||
fs.filer.DeleteChunks(context.WithoutCancel(ctx), fullPath, current.GetChunks())
|
||||
// Filer.UpdateEntry only writes the store; notify subscribers (sync,
|
||||
// backup, replication) of the new chunk ids like the gRPC path does.
|
||||
fs.filer.NotifyUpdateEvent(ctx, current, &newEntry, true, false, nil)
|
||||
writeJsonQuiet(w, r, http.StatusOK, map[string]interface{}{
|
||||
"name": entry.Name(), "size": size, "extents": len(layout.ExtentSizes),
|
||||
})
|
||||
}
|
||||
|
||||
// serveFormatView answers GET/HEAD /path?view=<adapter> for entries carrying a
|
||||
// layout. The layout is advisory: any inconsistency yields 404 here while the
|
||||
// plain read path stays untouched.
|
||||
func (fs *FilerServer) serveFormatView(ctx context.Context, w http.ResponseWriter, r *http.Request, entry *filer.Entry, viewName string) {
|
||||
adapter := format.ByName(viewName)
|
||||
viewer, viewerOk := adapter.(format.Viewer)
|
||||
if adapter == nil || !viewerOk {
|
||||
http.Error(w, "no such view", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
encoded := entry.Extended[format.LayoutKey]
|
||||
if len(encoded) == 0 {
|
||||
http.Error(w, "entry has no format layout", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
layout, err := format.DecodeLayout(encoded)
|
||||
if err != nil || layout.Format != viewName {
|
||||
http.Error(w, "entry has no such format layout", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// a size beyond int64 would read as negative, which Validate treats as
|
||||
// "skip the size check"
|
||||
if entry.FileSize > math.MaxInt64 {
|
||||
http.Error(w, "format layout is stale", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
if err := layout.Validate(int64(entry.FileSize)); err != nil {
|
||||
glog.WarningfCtx(ctx, "stale format layout on %s: %v", entry.FullPath, err)
|
||||
http.Error(w, "format layout is stale", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
// A write outside the format endpoints (offset writes, appends, mounts)
|
||||
// changes the chunks but keeps Extended, so the layout no longer
|
||||
// describes the bytes even when the total size still matches. Inline
|
||||
// content is disqualifying outright: format entries are never written
|
||||
// with it, and reads would prefer it over the chunks the layout maps.
|
||||
if len(entry.Content) != 0 || !bytes.Equal(entry.Extended[formatLayoutChunksKey], formatChunkIdentity(entry.GetChunks())) {
|
||||
glog.WarningfCtx(ctx, "format layout on %s no longer matches its bytes", entry.FullPath)
|
||||
http.Error(w, "format layout is stale", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// The view's validator must change when the layout or the requested
|
||||
// representation changes, even when the media bytes and their MD5 do not:
|
||||
// re-ingesting with a different sidecar must invalidate cached views.
|
||||
viewIdentity := md5.New()
|
||||
viewIdentity.Write([]byte(filer.ETagEntry(entry)))
|
||||
viewIdentity.Write(encoded)
|
||||
viewIdentity.Write([]byte(r.URL.RawQuery))
|
||||
viewEntry := *entry
|
||||
viewEntry.Md5 = viewIdentity.Sum(nil)
|
||||
if checkPreconditions(w, r, &viewEntry) {
|
||||
return
|
||||
}
|
||||
|
||||
plan, err := viewer.View(format.ViewRequest{Query: r.URL.Query()}, format.Object{
|
||||
Name: entry.Name(), Size: int64(entry.FileSize), Layout: layout,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, format.ErrNoSuchView) {
|
||||
http.NotFound(w, r)
|
||||
} else {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// pass through stored headers the way plain reads do
|
||||
for k, v := range entry.Extended {
|
||||
if !strings.HasPrefix(k, "xattr-") && !s3_constants.IsSeaweedFSInternalHeader(k) {
|
||||
w.Header().Set(k, string(v))
|
||||
}
|
||||
}
|
||||
// view responses are whole documents or whole extents
|
||||
w.Header().Set("Accept-Ranges", "none")
|
||||
w.Header().Set("Content-Type", plan.ContentType)
|
||||
SetEtag(w, filer.ETagEntry(&viewEntry))
|
||||
|
||||
if plan.Body != nil {
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(plan.Body)))
|
||||
if r.Method == http.MethodHead {
|
||||
return
|
||||
}
|
||||
if _, err := w.Write(plan.Body); err != nil {
|
||||
glog.V(2).InfofCtx(ctx, "write %s view of %s: %v", viewName, entry.FullPath, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
offset, extentSize, ok := layout.ExtentRange(plan.Extent)
|
||||
if !ok {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(extentSize, 10))
|
||||
if r.Method == http.MethodHead {
|
||||
return
|
||||
}
|
||||
streamCtx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
streamFn, err := filer.PrepareStreamContentWithPrefetch(streamCtx, fs.filer.MasterClient, fs.maybeGetVolumeReadJwtAuthorizationToken, entry.GetChunks(), offset, extentSize, fs.option.DownloadMaxBytesPs, filer.DefaultPrefetchCount)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if err := streamFn(w); err != nil {
|
||||
glog.ErrorfCtx(ctx, "stream %s view extent %d of %s: %v", viewName, plan.Extent, entry.FullPath, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,129 @@
|
||||
package weed_server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"math"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
)
|
||||
|
||||
func TestRoundUpToVolumeTTL(t *testing.T) {
|
||||
tests := []struct {
|
||||
seconds int64
|
||||
want int32
|
||||
}{
|
||||
{1, 60},
|
||||
{59, 60},
|
||||
{60, 60},
|
||||
{61, 120},
|
||||
{3599, 3600},
|
||||
{3600, 3600},
|
||||
{3601, 3660},
|
||||
{255 * 60, 255 * 60},
|
||||
{255*60 + 1, 5 * 3600}, // minutes overflow 255, ceil to hours
|
||||
{20_000_000, 232 * 24 * 3600}, // ~231.5 days, ceil to days
|
||||
{int64(math.MaxInt32), 0}, // beyond every unit's 255 cap: no TTL, never a shortened one
|
||||
}
|
||||
for _, test := range tests {
|
||||
got := roundUpToVolumeTTL(test.seconds)
|
||||
if got != test.want {
|
||||
t.Fatalf("roundUpToVolumeTTL(%d) = %d, want %d", test.seconds, got, test.want)
|
||||
}
|
||||
if got == 0 {
|
||||
continue // no volume TTL: chunks outlive the entry
|
||||
}
|
||||
if int64(got) < test.seconds {
|
||||
t.Fatalf("roundUpToVolumeTTL(%d) = %d shortened the lifetime", test.seconds, got)
|
||||
}
|
||||
// the rounded value must survive the volume TTL string conversion intact
|
||||
ttl, err := needle.ReadTTL(needle.SecondsToTTL(got))
|
||||
if err != nil || int64(ttl.Minutes())*60 != int64(got) {
|
||||
t.Fatalf("SecondsToTTL(%d) = %q does not round-trip (err %v)", got, needle.SecondsToTTL(got), err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepackSourceIdentity(t *testing.T) {
|
||||
base := func() *filer.Entry {
|
||||
return &filer.Entry{
|
||||
FullPath: "/videos/movie.ts",
|
||||
Attr: filer.Attr{
|
||||
FileSize: 30, TtlSec: 600,
|
||||
Crtime: time.Unix(1000, 0), Mtime: time.Unix(2000, 0),
|
||||
},
|
||||
Chunks: []*filer_pb.FileChunk{{FileId: "1,ab", Offset: 0, Size: 30}},
|
||||
}
|
||||
}
|
||||
identity := repackSourceIdentity(base())
|
||||
if !bytes.Equal(identity, repackSourceIdentity(base())) {
|
||||
t.Fatalf("identity is not deterministic")
|
||||
}
|
||||
mutations := map[string]func(*filer.Entry){
|
||||
"ttl cleared": func(e *filer.Entry) { e.TtlSec = 0 },
|
||||
"size changed": func(e *filer.Entry) { e.FileSize = 31 },
|
||||
"mtime moved": func(e *filer.Entry) { e.Mtime = time.Unix(3000, 0) },
|
||||
"crtime moved": func(e *filer.Entry) { e.Crtime = time.Unix(1001, 0) },
|
||||
"hard linked": func(e *filer.Entry) { e.HardLinkId = []byte{1} },
|
||||
"went remote": func(e *filer.Entry) { e.Remote = &filer_pb.RemoteEntry{} },
|
||||
"chunk moved": func(e *filer.Entry) { e.Chunks[0].Offset = 1 },
|
||||
"content set": func(e *filer.Entry) { e.Content = []byte{0x47} },
|
||||
}
|
||||
for name, mutate := range mutations {
|
||||
changed := base()
|
||||
mutate(changed)
|
||||
if bytes.Equal(identity, repackSourceIdentity(changed)) {
|
||||
t.Fatalf("identity ignored: %s", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatChunkIdentity(t *testing.T) {
|
||||
chunks := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "2,cd", Offset: 10, Size: 20},
|
||||
}
|
||||
identity := formatChunkIdentity(chunks)
|
||||
if !bytes.Equal(identity, formatChunkIdentity(chunks)) {
|
||||
t.Fatalf("identity is not deterministic")
|
||||
}
|
||||
changedFid := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "3,ef", Offset: 10, Size: 20},
|
||||
}
|
||||
if bytes.Equal(identity, formatChunkIdentity(changedFid)) {
|
||||
t.Fatalf("identity ignored a chunk replacement")
|
||||
}
|
||||
changedOffset := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "2,cd", Offset: 12, Size: 20},
|
||||
}
|
||||
if bytes.Equal(identity, formatChunkIdentity(changedOffset)) {
|
||||
t.Fatalf("identity ignored an offset change")
|
||||
}
|
||||
// a truncate mutates Size while keeping the chunk id
|
||||
changedSize := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "2,cd", Offset: 10, Size: 15},
|
||||
}
|
||||
if bytes.Equal(identity, formatChunkIdentity(changedSize)) {
|
||||
t.Fatalf("identity ignored a size change")
|
||||
}
|
||||
changedMtime := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "2,cd", Offset: 10, Size: 20, ModifiedTsNs: 7},
|
||||
}
|
||||
if bytes.Equal(identity, formatChunkIdentity(changedMtime)) {
|
||||
t.Fatalf("identity ignored a modification timestamp change")
|
||||
}
|
||||
changedManifest := []*filer_pb.FileChunk{
|
||||
{FileId: "1,ab", Offset: 0, Size: 10},
|
||||
{FileId: "2,cd", Offset: 10, Size: 20, IsChunkManifest: true},
|
||||
}
|
||||
if bytes.Equal(identity, formatChunkIdentity(changedManifest)) {
|
||||
t.Fatalf("identity ignored a manifest flag change")
|
||||
}
|
||||
}
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/format"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
||||
@@ -190,6 +191,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
if viewName := query.Get(format.ViewParam); viewName != "" {
|
||||
fs.serveFormatView(ctx, w, r, entry, viewName)
|
||||
return
|
||||
}
|
||||
|
||||
if checkPreconditions(w, r, entry) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -146,6 +146,14 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
|
||||
fs.move(ctx, w, r, so)
|
||||
} else if query.Has("cp.from") {
|
||||
fs.copy(ctx, w, r, so)
|
||||
} else if query.Get(formatIngestParam) != "" {
|
||||
if query.Get(formatRepackParam) != "" {
|
||||
writeJsonError(w, r, http.StatusBadRequest, errors.New(formatIngestParam+" and "+formatRepackParam+" are mutually exclusive"))
|
||||
return
|
||||
}
|
||||
fs.formatIngest(ctx, w, r, so)
|
||||
} else if query.Get(formatRepackParam) != "" {
|
||||
fs.formatRepack(ctx, w, r, so)
|
||||
} else if isExtractZip(r) {
|
||||
fs.extractZipArchive(ctx, w, r)
|
||||
} else {
|
||||
|
||||
@@ -274,6 +274,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
|
||||
// fix the path
|
||||
path := fs.fixFilePath(ctx, r, fileName)
|
||||
fullPath := util.FullPath(path)
|
||||
|
||||
var entry *filer.Entry
|
||||
var newChunks []*filer_pb.FileChunk
|
||||
@@ -296,11 +297,11 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
defer distributedLock.Stop()
|
||||
}
|
||||
if fs.entryLockTable != nil {
|
||||
pathLock := fs.entryLockTable.AcquireLock("appendEntry", util.FullPath(path), util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(util.FullPath(path), pathLock)
|
||||
pathLock := fs.entryLockTable.AcquireLock("appendEntry", fullPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(fullPath, pathLock)
|
||||
}
|
||||
var findErr error
|
||||
existingEntry, findErr = fs.filer.FindEntry(ctx, util.FullPath(path))
|
||||
existingEntry, findErr = fs.filer.FindEntry(ctx, fullPath)
|
||||
if findErr != nil && !errors.Is(findErr, filer_pb.ErrNotFound) {
|
||||
glog.V(0).InfofCtx(ctx, "failing to find %s: %v", path, findErr)
|
||||
replyerr = fmt.Errorf("find entry %q: %w", path, findErr)
|
||||
@@ -310,6 +311,20 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
|
||||
existingSnapshot = existingEntry.ToProtoEntry()
|
||||
entry = cloneEntryForAppend(existingEntry)
|
||||
}
|
||||
} else if fs.entryLockTable != nil {
|
||||
// Commit under the entry lock so plain HTTP overwrites serialize with
|
||||
// gRPC writers, renames, and format repack on this filer; cross-filer
|
||||
// serialization needs owner routing. The lock table is not reentrant,
|
||||
// so appends take it above under the append intention instead.
|
||||
pathLock := fs.entryLockTable.AcquireLock("saveMetaData", fullPath, util.ExclusiveLock)
|
||||
defer fs.entryLockTable.ReleaseLock(fullPath, pathLock)
|
||||
}
|
||||
|
||||
// recheck under the lock: WORM may have been enabled during the upload
|
||||
if enforced, wormErr := fs.wormEnforcedForEntry(ctx, path); wormErr != nil {
|
||||
return nil, wormErr, uncommittedChunks
|
||||
} else if enforced {
|
||||
return nil, errors.New(constants.ErrMsgOperationNotPermitted), uncommittedChunks
|
||||
}
|
||||
|
||||
inlineAppendHandled := false
|
||||
|
||||
@@ -30,6 +30,18 @@ var bufPool = sync.Pool{
|
||||
},
|
||||
}
|
||||
|
||||
// ChunkBoundaries decides where the upload loop may cut a storage chunk.
|
||||
type ChunkBoundaries interface {
|
||||
// NextChunkSize returns the size of the chunk starting at the absolute
|
||||
// file offset (startOffset included), or 0 when no further chunks are
|
||||
// expected. Returned sizes are bounded: they size in-memory buffers.
|
||||
NextChunkSize(offset int64) int64
|
||||
}
|
||||
|
||||
type fixedChunkSize int64
|
||||
|
||||
func (c fixedChunkSize) NextChunkSize(int64) int64 { return int64(c) }
|
||||
|
||||
func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.ResponseWriter, r *http.Request, reader io.Reader, chunkSize int32, fileName, contentType string, contentLength int64, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
|
||||
query := r.URL.Query()
|
||||
|
||||
@@ -102,7 +114,15 @@ func (fs *FilerServer) uploadRequestToChunks(ctx context.Context, w http.Respons
|
||||
}
|
||||
|
||||
func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, chunkSize int32, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
|
||||
return fs.uploadReaderToBoundedChunks(ctx, r, reader, startOffset, fixedChunkSize(chunkSize), fileName, contentType, isAppend, so)
|
||||
}
|
||||
|
||||
// uploadReaderToBoundedChunks cuts chunks where boundaries allows instead of
|
||||
// at a fixed size. The small-content optimization only applies to fixed-size
|
||||
// chunking: it must stay off when chunk boundaries carry meaning.
|
||||
func (fs *FilerServer) uploadReaderToBoundedChunks(ctx context.Context, r *http.Request, reader io.Reader, startOffset int64, boundaries ChunkBoundaries, fileName, contentType string, isAppend bool, so *operation.StorageOption) (fileChunks []*filer_pb.FileChunk, md5Hash hash.Hash, chunkOffset int64, uploadErr error, smallContent []byte) {
|
||||
|
||||
fixedSize, allowInline := boundaries.(fixedChunkSize)
|
||||
md5Hash = md5.New()
|
||||
chunkOffset = startOffset
|
||||
var partReader = io.NopCloser(io.TeeReader(reader, md5Hash))
|
||||
@@ -114,6 +134,11 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
|
||||
var uploadErrLock sync.Mutex
|
||||
for {
|
||||
|
||||
wantSize := boundaries.NextChunkSize(chunkOffset)
|
||||
if wantSize <= 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// need to throttle used byte buffer
|
||||
bytesBufferLimitChan <- struct{}{}
|
||||
|
||||
@@ -129,7 +154,7 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
|
||||
|
||||
bytesBuffer := bufPool.Get().(*bytes.Buffer)
|
||||
|
||||
limitedReader := io.LimitReader(partReader, int64(chunkSize))
|
||||
limitedReader := io.LimitReader(partReader, wantSize)
|
||||
|
||||
bytesBuffer.Reset()
|
||||
|
||||
@@ -148,10 +173,10 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
|
||||
}
|
||||
break
|
||||
}
|
||||
if chunkOffset == 0 && !isAppend {
|
||||
if chunkOffset == 0 && !isAppend && allowInline {
|
||||
// A short first read proves EOF, so the buffer holds the whole file.
|
||||
// A full chunk may have more data behind it — store it chunked.
|
||||
if dataSize < fs.option.SaveToFilerLimit && dataSize < int64(chunkSize) {
|
||||
if dataSize < fs.option.SaveToFilerLimit && dataSize < int64(fixedSize) {
|
||||
chunkOffset += dataSize
|
||||
smallContent = make([]byte, dataSize)
|
||||
bytesBuffer.Read(smallContent)
|
||||
@@ -194,7 +219,7 @@ func (fs *FilerServer) uploadReaderToChunks(ctx context.Context, r *http.Request
|
||||
chunkOffset = chunkOffset + dataSize
|
||||
|
||||
// if last chunk was not at full chunk size, but already exhausted the reader
|
||||
if dataSize < int64(chunkSize) {
|
||||
if dataSize < wantSize {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"sync"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
|
||||
@@ -34,9 +33,6 @@ func (c *commandVolumeConfigureReplication) Help() string {
|
||||
|
||||
This command changes a volume replication value. It should be followed by "volume.fix.replication".
|
||||
|
||||
Up to -maxParallelization volume servers are configured at a time, one volume at a time on each,
|
||||
and replicas of the same volume on different servers take turns.
|
||||
|
||||
`
|
||||
}
|
||||
|
||||
@@ -50,7 +46,6 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman
|
||||
volumeIdInt := configureReplicationCommand.Int("volumeId", 0, "the volume id")
|
||||
replicationString := configureReplicationCommand.String("replication", "", "the intended replication value")
|
||||
collectionPattern := configureReplicationCommand.String("collectionPattern", "", "comma-separated collection names, with '*' and '?' wildcards; empty matches all")
|
||||
maxParallelization := configureReplicationCommand.Int("maxParallelization", DefaultMaxParallelization, "run up to X tasks in parallel, whenever possible")
|
||||
if err = configureReplicationCommand.Parse(args); err != nil {
|
||||
return nil
|
||||
}
|
||||
@@ -80,19 +75,6 @@ func (c *commandVolumeConfigureReplication) Do(args []string, commandEnv *Comman
|
||||
return err
|
||||
}
|
||||
|
||||
return configureReplicationOnVolumeServers(commandEnv, topologyInfo, volumeFilter, replicaPlacement.String(), *maxParallelization)
|
||||
}
|
||||
|
||||
func configureReplicationOnVolumeServers(commandEnv *CommandEnv, topologyInfo *master_pb.TopologyInfo, volumeFilter func(*master_pb.VolumeInformationMessage) bool, replication string, maxParallelization int) error {
|
||||
type serverVolumes struct {
|
||||
addr pb.ServerAddress
|
||||
volumeIds []uint32
|
||||
}
|
||||
var servers []serverVolumes
|
||||
// VolumeConfigure unmounts and remounts the volume, so replicas take turns
|
||||
// to keep at least one of them serving.
|
||||
volumeLocks := make(map[uint32]*sync.Mutex)
|
||||
|
||||
// find all data nodes with volumes that needs replication change
|
||||
eachDataNode(topologyInfo, func(dc DataCenterId, rack RackId, dn *master_pb.DataNodeInfo) {
|
||||
var targetVolumeIds []uint32
|
||||
@@ -100,45 +82,33 @@ func configureReplicationOnVolumeServers(commandEnv *CommandEnv, topologyInfo *m
|
||||
for _, v := range diskInfo.VolumeInfos {
|
||||
if volumeFilter(v) {
|
||||
targetVolumeIds = append(targetVolumeIds, v.Id)
|
||||
if volumeLocks[v.Id] == nil {
|
||||
volumeLocks[v.Id] = &sync.Mutex{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(targetVolumeIds) > 0 {
|
||||
servers = append(servers, serverVolumes{pb.NewServerAddressFromDataNode(dn), targetVolumeIds})
|
||||
if len(targetVolumeIds) == 0 {
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
ewg := NewErrorWaitGroup(maxParallelization)
|
||||
for _, server := range servers {
|
||||
ewg.Add(func() error {
|
||||
err := operation.WithVolumeServerClient(false, server.addr, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
for _, targetVolumeId := range server.volumeIds {
|
||||
volumeLocks[targetVolumeId].Lock()
|
||||
resp, configureErr := volumeServerClient.VolumeConfigure(context.Background(), &volume_server_pb.VolumeConfigureRequest{
|
||||
VolumeId: targetVolumeId,
|
||||
Replication: replication,
|
||||
})
|
||||
volumeLocks[targetVolumeId].Unlock()
|
||||
if configureErr != nil {
|
||||
return configureErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return errors.New(resp.Error)
|
||||
}
|
||||
err = operation.WithVolumeServerClient(false, pb.NewServerAddressFromDataNode(dn), commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
|
||||
for _, targetVolumeId := range targetVolumeIds {
|
||||
resp, configureErr := volumeServerClient.VolumeConfigure(context.Background(), &volume_server_pb.VolumeConfigureRequest{
|
||||
VolumeId: targetVolumeId,
|
||||
Replication: replicaPlacement.String(),
|
||||
})
|
||||
if configureErr != nil {
|
||||
return configureErr
|
||||
}
|
||||
if resp.Error != "" {
|
||||
return errors.New(resp.Error)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("volume server %s: %w", server.addr, err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
return ewg.Wait()
|
||||
return err
|
||||
}
|
||||
|
||||
func getVolumeFilter(replicaPlacement *super_block.ReplicaPlacement, volumeId uint32, collectionPattern string) (func(message *master_pb.VolumeInformationMessage) bool, error) {
|
||||
|
||||
@@ -1,226 +0,0 @@
|
||||
package shell
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"maps"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
|
||||
type configureReplicationTestVolumeServer struct {
|
||||
volume_server_pb.UnimplementedVolumeServerServer
|
||||
failure string
|
||||
mu sync.Mutex
|
||||
configured map[uint32]string
|
||||
}
|
||||
|
||||
func (s *configureReplicationTestVolumeServer) VolumeConfigure(_ context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.configured == nil {
|
||||
s.configured = make(map[uint32]string)
|
||||
}
|
||||
s.configured[req.VolumeId] = req.Replication
|
||||
return &volume_server_pb.VolumeConfigureResponse{Error: s.failure}, nil
|
||||
}
|
||||
|
||||
// configureReplicationOverlapServer can back several volume servers at once and
|
||||
// records whether two calls for the same volume were ever in flight together.
|
||||
type configureReplicationOverlapServer struct {
|
||||
volume_server_pb.UnimplementedVolumeServerServer
|
||||
mu sync.Mutex
|
||||
inFlight map[uint32]int
|
||||
overlapped bool
|
||||
}
|
||||
|
||||
func (s *configureReplicationOverlapServer) VolumeConfigure(_ context.Context, req *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
|
||||
s.mu.Lock()
|
||||
s.inFlight[req.VolumeId]++
|
||||
if s.inFlight[req.VolumeId] > 1 {
|
||||
s.overlapped = true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
|
||||
s.mu.Lock()
|
||||
s.inFlight[req.VolumeId]--
|
||||
s.mu.Unlock()
|
||||
return &volume_server_pb.VolumeConfigureResponse{}, nil
|
||||
}
|
||||
|
||||
func startConfigureReplicationTestVolumeServer(t *testing.T, server volume_server_pb.VolumeServerServer, volumeIds ...uint32) *master_pb.DataNodeInfo {
|
||||
t.Helper()
|
||||
lis, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
srv := grpc.NewServer()
|
||||
volume_server_pb.RegisterVolumeServerServer(srv, server)
|
||||
go srv.Serve(lis)
|
||||
t.Cleanup(srv.Stop)
|
||||
|
||||
port := lis.Addr().(*net.TCPAddr).Port
|
||||
disk := &master_pb.DiskInfo{}
|
||||
for _, vid := range volumeIds {
|
||||
disk.VolumeInfos = append(disk.VolumeInfos, &master_pb.VolumeInformationMessage{Id: vid})
|
||||
}
|
||||
return &master_pb.DataNodeInfo{
|
||||
Id: fmt.Sprintf("127.0.0.1:%d", port),
|
||||
GrpcPort: uint32(port),
|
||||
DiskInfos: map[string]*master_pb.DiskInfo{"": disk},
|
||||
}
|
||||
}
|
||||
|
||||
func configureReplicationTestEnv(dataNodes ...*master_pb.DataNodeInfo) (*CommandEnv, *master_pb.TopologyInfo) {
|
||||
commandEnv := &CommandEnv{option: &ShellOptions{
|
||||
GrpcDialOption: grpc.WithTransportCredentials(insecure.NewCredentials()),
|
||||
}}
|
||||
topologyInfo := &master_pb.TopologyInfo{DataCenterInfos: []*master_pb.DataCenterInfo{{
|
||||
RackInfos: []*master_pb.RackInfo{{DataNodeInfos: dataNodes}},
|
||||
}}}
|
||||
return commandEnv, topologyInfo
|
||||
}
|
||||
|
||||
func allVolumes(*master_pb.VolumeInformationMessage) bool { return true }
|
||||
|
||||
func TestConfigureReplicationOnVolumeServersReportsEveryFailingServer(t *testing.T) {
|
||||
for _, maxParallelization := range []int{1, DefaultMaxParallelization} {
|
||||
t.Run(fmt.Sprintf("maxParallelization=%d", maxParallelization), func(t *testing.T) {
|
||||
first := &configureReplicationTestVolumeServer{failure: "first server failed"}
|
||||
healthy := &configureReplicationTestVolumeServer{}
|
||||
last := &configureReplicationTestVolumeServer{failure: "last server failed"}
|
||||
commandEnv, topologyInfo := configureReplicationTestEnv(
|
||||
startConfigureReplicationTestVolumeServer(t, first, 1),
|
||||
startConfigureReplicationTestVolumeServer(t, healthy, 2, 3),
|
||||
startConfigureReplicationTestVolumeServer(t, last, 4),
|
||||
)
|
||||
|
||||
err := configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "001", maxParallelization)
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
for _, failure := range []string{first.failure, last.failure} {
|
||||
if !strings.Contains(err.Error(), failure) {
|
||||
t.Errorf("error %q does not mention %q", err, failure)
|
||||
}
|
||||
}
|
||||
healthy.mu.Lock()
|
||||
defer healthy.mu.Unlock()
|
||||
if expected := map[uint32]string{2: "001", 3: "001"}; !maps.Equal(healthy.configured, expected) {
|
||||
t.Errorf("healthy server configured %v, expected %v", healthy.configured, expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigureReplicationOnVolumeServersConfiguresReplicasOneAtATime(t *testing.T) {
|
||||
server := &configureReplicationOverlapServer{inFlight: make(map[uint32]int)}
|
||||
commandEnv, topologyInfo := configureReplicationTestEnv(
|
||||
startConfigureReplicationTestVolumeServer(t, server, 1, 2),
|
||||
startConfigureReplicationTestVolumeServer(t, server, 1, 2),
|
||||
startConfigureReplicationTestVolumeServer(t, server, 1, 2),
|
||||
)
|
||||
|
||||
if err := configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "002", DefaultMaxParallelization); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
server.mu.Lock()
|
||||
defer server.mu.Unlock()
|
||||
if server.overlapped {
|
||||
t.Error("replicas of one volume were configured at the same time")
|
||||
}
|
||||
}
|
||||
|
||||
// configureReplicationGateServer blocks inside VolumeConfigure until the test
|
||||
// releases it, so the test can observe how many servers run at once.
|
||||
type configureReplicationGateServer struct {
|
||||
volume_server_pb.UnimplementedVolumeServerServer
|
||||
gate *configureReplicationGate
|
||||
}
|
||||
|
||||
type configureReplicationGate struct {
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
mu sync.Mutex
|
||||
active int
|
||||
maxActive int
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *configureReplicationGateServer) VolumeConfigure(_ context.Context, _ *volume_server_pb.VolumeConfigureRequest) (*volume_server_pb.VolumeConfigureResponse, error) {
|
||||
s.gate.mu.Lock()
|
||||
s.gate.calls++
|
||||
s.gate.active++
|
||||
if s.gate.active > s.gate.maxActive {
|
||||
s.gate.maxActive = s.gate.active
|
||||
}
|
||||
s.gate.mu.Unlock()
|
||||
|
||||
s.gate.entered <- struct{}{}
|
||||
<-s.gate.release
|
||||
|
||||
s.gate.mu.Lock()
|
||||
s.gate.active--
|
||||
s.gate.mu.Unlock()
|
||||
return &volume_server_pb.VolumeConfigureResponse{}, nil
|
||||
}
|
||||
|
||||
func TestConfigureReplicationOnVolumeServersRunsServersInParallelUpToTheLimit(t *testing.T) {
|
||||
const servers = 4
|
||||
const maxParallelization = 2
|
||||
gate := &configureReplicationGate{
|
||||
entered: make(chan struct{}, servers),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
var dataNodes []*master_pb.DataNodeInfo
|
||||
for i := 0; i < servers; i++ {
|
||||
dataNodes = append(dataNodes, startConfigureReplicationTestVolumeServer(t,
|
||||
&configureReplicationGateServer{gate: gate}, uint32(i+1)))
|
||||
}
|
||||
commandEnv, topologyInfo := configureReplicationTestEnv(dataNodes...)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
done <- configureReplicationOnVolumeServers(commandEnv, topologyInfo, allVolumes, "001", maxParallelization)
|
||||
}()
|
||||
|
||||
for i := 0; i < maxParallelization; i++ {
|
||||
select {
|
||||
case <-gate.entered:
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("only %d servers started; serial execution would hold the second call", i)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-gate.entered:
|
||||
t.Fatalf("more than %d servers ran while calls were held open", maxParallelization)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
}
|
||||
|
||||
close(gate.release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gate.mu.Lock()
|
||||
defer gate.mu.Unlock()
|
||||
if gate.maxActive != maxParallelization {
|
||||
t.Errorf("at most %d servers ran together, expected %d", gate.maxActive, maxParallelization)
|
||||
}
|
||||
if gate.calls != servers {
|
||||
t.Errorf("%d servers ran, expected %d", gate.calls, servers)
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,6 @@ import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -47,8 +46,8 @@ func ReplicatedWrite(ctx context.Context, masterFn operation.GetMasterFn, grpcDi
|
||||
}
|
||||
|
||||
// with no local volume and no replica to forward to, the write would be acknowledged without being stored
|
||||
vol := s.GetVolume(volumeId)
|
||||
if vol == nil && len(remoteLocations) == 0 {
|
||||
hasLocalVolume := s.HasVolume(volumeId)
|
||||
if !hasLocalVolume && len(remoteLocations) == 0 {
|
||||
err = fmt.Errorf("volume %d not found on %s:%d", volumeId, s.Ip, s.Port)
|
||||
glog.V(0).Infoln(err)
|
||||
return
|
||||
@@ -69,141 +68,109 @@ func ReplicatedWrite(ctx context.Context, masterFn operation.GetMasterFn, grpcDi
|
||||
}(time.Now())
|
||||
}
|
||||
|
||||
// Snapshot the fields the fan-out reads before the concurrent local write
|
||||
// mutates the needle (it stamps the volume TTL and flags), so the two cannot
|
||||
// race; the fan-out then touches only these locals plus read-only n.Data.
|
||||
replicaTtl := n.Ttl
|
||||
if vol != nil && replicaTtl == needle.EMPTY_TTL && vol.Ttl != needle.EMPTY_TTL {
|
||||
replicaTtl = vol.Ttl
|
||||
}
|
||||
replicaLastModified := n.LastModified
|
||||
replicaIsChunked := n.IsChunkedManifest()
|
||||
replicaIsCompressed := n.IsCompressed()
|
||||
replicaName := string(n.Name)
|
||||
replicaMime := string(n.Mime)
|
||||
replicaData := n.Data
|
||||
replicaPairMap := make(map[string]string)
|
||||
if replicaCount > 0 && n.HasPairs() {
|
||||
tmpMap := make(map[string]string)
|
||||
if pe := json.Unmarshal(n.Pairs, &tmpMap); pe != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
|
||||
glog.V(0).Infoln("Unmarshal pairs error:", pe)
|
||||
}
|
||||
for k, v := range tmpMap {
|
||||
replicaPairMap[needle.PairNamePrefix+k] = v
|
||||
if hasLocalVolume {
|
||||
start := time.Now()
|
||||
|
||||
inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToLocalDisk)
|
||||
inFlightGauge.Inc()
|
||||
defer inFlightGauge.Dec()
|
||||
|
||||
isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, true, fsync)
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
|
||||
err = fmt.Errorf("failed to write to local disk: %w", err)
|
||||
glog.V(0).Infoln(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Observe replication targets histogram for all operations (including zero)
|
||||
stats.VolumeServerReplicationTargets.Observe(float64(replicaCount))
|
||||
|
||||
// Run the local disk write and the replica fan-out concurrently against the
|
||||
// buffered needle (latency max(local, replicas), not the sum); ack after both.
|
||||
var wg sync.WaitGroup
|
||||
var localErr, remoteErr error
|
||||
var localIsUnchanged bool
|
||||
if replicaCount > 0 { //send to other replica locations
|
||||
start := time.Now()
|
||||
|
||||
if vol != nil {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
start := time.Now()
|
||||
inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToLocalDisk)
|
||||
inFlightGauge.Inc()
|
||||
defer inFlightGauge.Dec()
|
||||
inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToReplicas)
|
||||
inFlightGauge.Inc()
|
||||
defer inFlightGauge.Dec()
|
||||
|
||||
localIsUnchanged, localErr = s.WriteVolumeNeedle(volumeId, n, true, fsync)
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToLocalDisk).Observe(time.Since(start).Seconds())
|
||||
if localErr != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToLocalDisk).Inc()
|
||||
localErr = fmt.Errorf("failed to write to local disk: %w", localErr)
|
||||
glog.V(0).Infoln(localErr)
|
||||
err = DistributedOperation(ctx, remoteLocations, func(ctx context.Context, location operation.Location) error {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: location.Url,
|
||||
Path: r.URL.Path,
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if replicaCount > 0 {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
start := time.Now()
|
||||
inFlightGauge := stats.VolumeServerInFlightRequestsGauge.WithLabelValues(stats.WriteToReplicas)
|
||||
inFlightGauge.Inc()
|
||||
defer inFlightGauge.Dec()
|
||||
|
||||
remoteErr = DistributedOperation(ctx, remoteLocations, func(ctx context.Context, location operation.Location) error {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: location.Url,
|
||||
Path: r.URL.Path,
|
||||
}
|
||||
q := url.Values{
|
||||
"type": {"replicate"},
|
||||
"ttl": {replicaTtl.String()},
|
||||
}
|
||||
if fsync {
|
||||
q.Set("fsync", "true")
|
||||
}
|
||||
if replicaLastModified > 0 {
|
||||
q.Set("ts", strconv.FormatUint(replicaLastModified, 10))
|
||||
}
|
||||
if replicaIsChunked {
|
||||
q.Set("cm", "true")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
bytesBuffer := buffer_pool.SyncPoolGetBuffer()
|
||||
defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
|
||||
|
||||
// volume server do not know about encryption
|
||||
uploadOption := &operation.UploadOption{
|
||||
UploadUrl: u.String(),
|
||||
Filename: replicaName,
|
||||
Cipher: false,
|
||||
IsInputCompressed: replicaIsCompressed,
|
||||
IsReplication: true,
|
||||
MimeType: replicaMime,
|
||||
PairMap: replicaPairMap,
|
||||
Jwt: jwt,
|
||||
Md5: contentMd5,
|
||||
BytesBuffer: bytesBuffer,
|
||||
MaxAttempts: 1, // fail fast on a dead replica; the client write retries
|
||||
}
|
||||
|
||||
uploader, uploaderErr := operation.NewUploader()
|
||||
if uploaderErr != nil {
|
||||
glog.Errorf("replication-UploadData, err:%v, url:%s", uploaderErr, u.String())
|
||||
return uploaderErr
|
||||
}
|
||||
_, uploadErr := uploader.UploadData(ctx, replicaData, uploadOption)
|
||||
if uploadErr != nil {
|
||||
glog.Errorf("replication-UploadData, err:%v, url:%s", uploadErr, u.String())
|
||||
}
|
||||
return uploadErr
|
||||
})
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds())
|
||||
if remoteErr != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
|
||||
stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationFailure).Inc()
|
||||
reason := classifyReplicationError(remoteErr)
|
||||
stats.VolumeServerReplicationFailures.WithLabelValues(stats.ReplicationOpWrite, reason).Inc()
|
||||
remoteErr = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, remoteErr)
|
||||
glog.V(0).Infoln(remoteErr)
|
||||
} else {
|
||||
stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationSuccess).Inc()
|
||||
q := url.Values{
|
||||
"type": {"replicate"},
|
||||
"ttl": {n.Ttl.String()},
|
||||
}
|
||||
}()
|
||||
}
|
||||
if fsync {
|
||||
q.Set("fsync", "true")
|
||||
}
|
||||
if n.LastModified > 0 {
|
||||
q.Set("ts", strconv.FormatUint(n.LastModified, 10))
|
||||
}
|
||||
if n.IsChunkedManifest() {
|
||||
q.Set("cm", "true")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
|
||||
wg.Wait()
|
||||
pairMap := make(map[string]string)
|
||||
if n.HasPairs() {
|
||||
tmpMap := make(map[string]string)
|
||||
err := json.Unmarshal(n.Pairs, &tmpMap)
|
||||
if err != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorUnmarshalPairs).Inc()
|
||||
glog.V(0).Infoln("Unmarshal pairs error:", err)
|
||||
}
|
||||
for k, v := range tmpMap {
|
||||
pairMap[needle.PairNamePrefix+k] = v
|
||||
}
|
||||
}
|
||||
bytesBuffer := buffer_pool.SyncPoolGetBuffer()
|
||||
defer buffer_pool.SyncPoolPutBuffer(bytesBuffer)
|
||||
|
||||
if localErr != nil {
|
||||
return false, localErr
|
||||
// volume server do not know about encryption
|
||||
// TODO optimize here to compress data only once
|
||||
uploadOption := &operation.UploadOption{
|
||||
UploadUrl: u.String(),
|
||||
Filename: string(n.Name),
|
||||
Cipher: false,
|
||||
IsInputCompressed: n.IsCompressed(),
|
||||
IsReplication: true,
|
||||
MimeType: string(n.Mime),
|
||||
PairMap: pairMap,
|
||||
Jwt: jwt,
|
||||
Md5: contentMd5,
|
||||
BytesBuffer: bytesBuffer,
|
||||
MaxAttempts: 1, // fail fast on a dead replica; the client write retries
|
||||
}
|
||||
|
||||
uploader, err := operation.NewUploader()
|
||||
if err != nil {
|
||||
glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String())
|
||||
return err
|
||||
}
|
||||
_, err = uploader.UploadData(ctx, n.Data, uploadOption)
|
||||
if err != nil {
|
||||
glog.Errorf("replication-UploadData, err:%v, url:%s", err, u.String())
|
||||
}
|
||||
return err
|
||||
})
|
||||
stats.VolumeServerRequestHistogram.WithLabelValues(stats.WriteToReplicas).Observe(time.Since(start).Seconds())
|
||||
if err != nil {
|
||||
stats.VolumeServerHandlerCounter.WithLabelValues(stats.ErrorWriteToReplicas).Inc()
|
||||
stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationFailure).Inc()
|
||||
reason := classifyReplicationError(err)
|
||||
stats.VolumeServerReplicationFailures.WithLabelValues(stats.ReplicationOpWrite, reason).Inc()
|
||||
err = fmt.Errorf("failed to write to replicas for volume %d: %v", volumeId, err)
|
||||
glog.V(0).Infoln(err)
|
||||
return false, err
|
||||
}
|
||||
stats.VolumeServerReplicationCounter.WithLabelValues(stats.ReplicationOpWrite, stats.ReplicationSuccess).Inc()
|
||||
}
|
||||
if remoteErr != nil {
|
||||
return false, remoteErr
|
||||
}
|
||||
return localIsUnchanged, nil
|
||||
return
|
||||
}
|
||||
|
||||
// ReplicatedDelete deletes a needle from the local volume and sends delete
|
||||
|
||||
@@ -16,11 +16,8 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
)
|
||||
@@ -233,176 +230,6 @@ func TestReplicatedWriteForwardsFsyncToReplicas(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// newReplicateTestStore builds a single-disk store in dir, draining the notify
|
||||
// channels so volume mounts never block on a heartbeat reader.
|
||||
func newReplicateTestStore(t *testing.T, dir string) *storage.Store {
|
||||
t.Helper()
|
||||
store := storage.NewStore(nil, "127.0.0.1", 8080, 18080, "http://127.0.0.1:8080", "store-id",
|
||||
[]string{dir}, []int32{100}, []util.MinFreeSpace{{}}, "",
|
||||
storage.NeedleMapInMemory, []types.DiskType{types.HardDriveType}, nil, 3,
|
||||
stats.DefaultDiskIOProbeConfig())
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-store.NewVolumesChan:
|
||||
case <-store.DeletedVolumesChan:
|
||||
case <-store.NewEcShardsChan:
|
||||
case <-store.DeletedEcShardsChan:
|
||||
case <-store.StateUpdateChan:
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
t.Cleanup(func() {
|
||||
close(done)
|
||||
store.Close()
|
||||
})
|
||||
return store
|
||||
}
|
||||
|
||||
// TestReplicatedWriteOverlapsLocalAndReplicaWrites runs the write path with a
|
||||
// real local volume and a replica that stalls mid-request: the fan-out starts
|
||||
// while the local write is still free to run, the local write lands before the
|
||||
// replica unblocks, and the acknowledged write waits for both legs.
|
||||
func TestReplicatedWriteOverlapsLocalAndReplicaWrites(t *testing.T) {
|
||||
replicaStarted := make(chan url.Values, 1)
|
||||
releaseReplica := make(chan struct{})
|
||||
var releaseOnce sync.Once
|
||||
release := func() { releaseOnce.Do(func() { close(releaseReplica) }) }
|
||||
replica := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
replicaStarted <- r.URL.Query()
|
||||
<-releaseReplica
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
if _, err := w.Write([]byte(`{"size":21}`)); err != nil {
|
||||
t.Errorf("replica write response: %v", err)
|
||||
}
|
||||
}))
|
||||
defer replica.Close()
|
||||
defer release()
|
||||
replicaHost := strings.TrimPrefix(replica.URL, "http://")
|
||||
|
||||
masterFn, dialOption := startMockMasterServer(t, &mockMasterServer{
|
||||
locations: []*master_pb.Location{{Url: "127.0.0.1:8080"}, {Url: replicaHost}},
|
||||
})
|
||||
|
||||
store := newReplicateTestStore(t, t.TempDir())
|
||||
volumeId := needle.VolumeId(7)
|
||||
operation.InvalidateVolumeIdLocationCache(volumeId.String())
|
||||
if err := store.AddVolume(volumeId, "", storage.NeedleMapInMemory, "001", "15d", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
data := []byte("overlapped write data")
|
||||
n := &needle.Needle{
|
||||
Id: 1,
|
||||
Data: data,
|
||||
Checksum: needle.NewCRC(data),
|
||||
Ttl: needle.EMPTY_TTL,
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, "http://127.0.0.1:8080/7,01637037d6", nil)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := ReplicatedWrite(context.Background(), masterFn, dialOption, store, volumeId, n, r, "")
|
||||
done <- err
|
||||
}()
|
||||
|
||||
var q url.Values
|
||||
select {
|
||||
case q = <-replicaStarted:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("replica never received the fan-out request")
|
||||
}
|
||||
if got := q.Get("type"); got != "replicate" {
|
||||
t.Errorf("replica type = %q, want replicate", got)
|
||||
}
|
||||
if got := q.Get("ttl"); got != "15d" {
|
||||
t.Errorf("replica ttl = %q, want the inherited volume ttl 15d", got)
|
||||
}
|
||||
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("write returned before the replica responded: %v", err)
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
readBack := &needle.Needle{Id: n.Id}
|
||||
if _, err := store.ReadVolumeNeedle(volumeId, readBack, nil, nil); err == nil && string(readBack.Data) == string(data) {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("local write did not land while the replica was still blocked")
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
|
||||
release()
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("ReplicatedWrite: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicatedWriteFailsWhenEitherLegFails checks that a failed local write
|
||||
// or a failed replica fan-out fails the whole request even when the other leg
|
||||
// succeeded.
|
||||
func TestReplicatedWriteFailsWhenEitherLegFails(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
volumeId needle.VolumeId
|
||||
readonlyLocal bool
|
||||
replicaStatus int
|
||||
}{
|
||||
{name: "local write fails", volumeId: 8, readonlyLocal: true, replicaStatus: http.StatusCreated},
|
||||
{name: "replica write fails", volumeId: 9, replicaStatus: http.StatusInternalServerError},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
replica := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(tc.replicaStatus)
|
||||
}))
|
||||
defer replica.Close()
|
||||
replicaHost := strings.TrimPrefix(replica.URL, "http://")
|
||||
|
||||
masterFn, dialOption := startMockMasterServer(t, &mockMasterServer{
|
||||
locations: []*master_pb.Location{{Url: "127.0.0.1:8080"}, {Url: replicaHost}},
|
||||
})
|
||||
|
||||
store := newReplicateTestStore(t, t.TempDir())
|
||||
operation.InvalidateVolumeIdLocationCache(tc.volumeId.String())
|
||||
if err := store.AddVolume(tc.volumeId, "", storage.NeedleMapInMemory, "001", "", 0, needle.GetCurrentVersion(), 0, types.HardDriveType, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if tc.readonlyLocal {
|
||||
if err := store.MarkVolumeReadonly(tc.volumeId, false, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
data := []byte("write data")
|
||||
n := &needle.Needle{
|
||||
Id: 1,
|
||||
Data: data,
|
||||
Checksum: needle.NewCRC(data),
|
||||
Ttl: needle.EMPTY_TTL,
|
||||
}
|
||||
r := httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://127.0.0.1:8080/%d,01637037d6", tc.volumeId), nil)
|
||||
if err := r.ParseForm(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err := ReplicatedWrite(context.Background(), masterFn, dialOption, store, tc.volumeId, n, r, ""); err == nil {
|
||||
t.Fatal("ReplicatedWrite succeeded despite a failed leg")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicatedWriteRejectsWriteWithNoTarget verifies that a write this server
|
||||
// cannot store and has no replica to forward to fails instead of being
|
||||
// acknowledged with nothing written.
|
||||
|
||||
@@ -165,7 +165,7 @@ func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipLis
|
||||
Timeout: timeout,
|
||||
}
|
||||
|
||||
if host != "localhost" && host != "" && host != "0.0.0.0" && host != "127.0.0.1" && host != "[::]" && host != "[::1]" && !isLoopbackHost(host) {
|
||||
if host != "localhost" && host != "" && host != "0.0.0.0" && host != "127.0.0.1" && host != "[::]" && host != "[::1]" {
|
||||
listener, err = net.Listen("tcp", JoinHostPort("localhost", port))
|
||||
if err != nil {
|
||||
glog.V(0).Infof("skip starting on %s:%d: %v", host, port, err)
|
||||
|
||||
@@ -20,9 +20,8 @@ var (
|
||||
)
|
||||
|
||||
// SetOutboundLocalIP records ip as the source address for outbound TCP
|
||||
// connections. An empty, wildcard (0.0.0.0 / ::), loopback, or unparseable
|
||||
// value leaves the source address to the OS. A loopback source can only
|
||||
// originate connections to loopback targets, which already skip binding.
|
||||
// connections. An empty, wildcard (0.0.0.0 / ::), or unparseable value leaves
|
||||
// the source address to the OS.
|
||||
//
|
||||
// The first call wins. `weed server` applies the top-level -ip.bind before its
|
||||
// embedded master/volume/filer/s3/... start, so a component's own (possibly
|
||||
@@ -33,7 +32,7 @@ func SetOutboundLocalIP(ip string) {
|
||||
return
|
||||
}
|
||||
parsed := net.ParseIP(ip)
|
||||
if parsed == nil || parsed.IsUnspecified() || parsed.IsLoopback() {
|
||||
if parsed == nil || parsed.IsUnspecified() {
|
||||
return
|
||||
}
|
||||
outboundLocalAddr.Store(&net.TCPAddr{IP: parsed})
|
||||
|
||||
@@ -24,8 +24,6 @@ func TestSetOutboundLocalIP(t *testing.T) {
|
||||
{"empty", "", ""},
|
||||
{"wildcard v4", "0.0.0.0", ""},
|
||||
{"wildcard v6", "::", ""},
|
||||
{"loopback v4", "127.0.0.2", ""},
|
||||
{"loopback v6", "::1", ""},
|
||||
{"garbage", "not-an-ip", ""},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// WrapProxyProtocolListener returns a listener whose accepted connections
|
||||
// read and strip a HAProxy PROXY protocol v1 header line, exposing the
|
||||
// advertised source address as the connection's RemoteAddr. Connections
|
||||
// that do not start with a PROXY header keep their real remote address,
|
||||
// so direct clients continue to work (and fail any address-based checks
|
||||
// against their actual address).
|
||||
//
|
||||
// The header is parsed lazily on the first Read or RemoteAddr call, inside
|
||||
// the connection's own read path; a silent or malformed peer occupies only
|
||||
// its own connection, never the accept loop.
|
||||
//
|
||||
// Only enable this on listeners reachable solely through a trusted proxy:
|
||||
// any direct client could otherwise spoof its reported address.
|
||||
func WrapProxyProtocolListener(listener net.Listener) net.Listener {
|
||||
return &proxyProtocolListener{Listener: listener}
|
||||
}
|
||||
|
||||
type proxyProtocolListener struct {
|
||||
net.Listener
|
||||
}
|
||||
|
||||
func (l *proxyProtocolListener) Accept() (net.Conn, error) {
|
||||
conn, err := l.Listener.Accept()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &proxyProtocolConn{Conn: conn}, nil
|
||||
}
|
||||
|
||||
type proxyProtocolConn struct {
|
||||
net.Conn
|
||||
parseOnce sync.Once
|
||||
reader *bufio.Reader
|
||||
remote net.Addr
|
||||
headerErr error
|
||||
}
|
||||
|
||||
func (c *proxyProtocolConn) Read(b []byte) (int, error) {
|
||||
c.parse()
|
||||
if c.headerErr != nil {
|
||||
return 0, c.headerErr
|
||||
}
|
||||
return c.reader.Read(b)
|
||||
}
|
||||
|
||||
func (c *proxyProtocolConn) RemoteAddr() net.Addr {
|
||||
c.parse()
|
||||
if c.remote != nil {
|
||||
return c.remote
|
||||
}
|
||||
return c.Conn.RemoteAddr()
|
||||
}
|
||||
|
||||
func (c *proxyProtocolConn) parse() {
|
||||
c.parseOnce.Do(func() {
|
||||
c.reader = bufio.NewReader(c.Conn)
|
||||
head, err := c.reader.Peek(len("PROXY "))
|
||||
if err != nil {
|
||||
if err != io.EOF {
|
||||
c.headerErr = err
|
||||
}
|
||||
return
|
||||
}
|
||||
if string(head) != "PROXY " {
|
||||
return
|
||||
}
|
||||
line, err := c.reader.ReadSlice('\n')
|
||||
if err != nil {
|
||||
// A truncated or overlong header means the peer never sent a
|
||||
// parseable line; keep the real address and let the buffered
|
||||
// bytes fail the downstream protocol on their own.
|
||||
c.discardHeaderLine()
|
||||
return
|
||||
}
|
||||
fields := strings.Fields(strings.TrimRight(string(line), "\r\n"))
|
||||
if len(fields) == 6 && fields[0] == "PROXY" && (fields[1] == "TCP4" || fields[1] == "TCP6") {
|
||||
if srcIP := net.ParseIP(fields[2]); srcIP != nil {
|
||||
if srcPort, err := strconv.Atoi(fields[4]); err == nil {
|
||||
c.remote = &net.TCPAddr{IP: srcIP, Port: srcPort}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// discardHeaderLine drops the remainder of a malformed PROXY line so the
|
||||
// following bytes still reach the wrapped connection.
|
||||
func (c *proxyProtocolConn) discardHeaderLine() {
|
||||
discard := make([]byte, c.reader.Buffered())
|
||||
_, _ = io.ReadFull(c.reader, discard)
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"io"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func acceptOne(t *testing.T, ln net.Listener) net.Conn {
|
||||
t.Helper()
|
||||
accept := make(chan net.Conn, 1)
|
||||
go func() {
|
||||
conn, err := ln.Accept()
|
||||
if err == nil {
|
||||
accept <- conn
|
||||
}
|
||||
}()
|
||||
select {
|
||||
case conn := <-accept:
|
||||
return conn
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("Accept blocked waiting for the peer to send data")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newPipePair(t *testing.T) (client, server net.Conn) {
|
||||
t.Helper()
|
||||
serverConn, client := net.Pipe()
|
||||
ln := WrapProxyProtocolListener(&oneShotListener{conn: serverConn})
|
||||
return client, acceptOne(t, ln)
|
||||
}
|
||||
|
||||
type oneShotListener struct {
|
||||
conn net.Conn
|
||||
done bool
|
||||
}
|
||||
|
||||
func (l *oneShotListener) Accept() (net.Conn, error) {
|
||||
if l.done {
|
||||
return nil, io.EOF
|
||||
}
|
||||
l.done = true
|
||||
return l.conn, nil
|
||||
}
|
||||
func (l *oneShotListener) Close() error { return l.conn.Close() }
|
||||
func (l *oneShotListener) Addr() net.Addr { return l.conn.LocalAddr() }
|
||||
|
||||
func TestProxyProtocolHeaderRewritesRemoteAddr(t *testing.T) {
|
||||
client, server := newPipePair(t)
|
||||
defer client.Close()
|
||||
defer server.Close()
|
||||
|
||||
go func() {
|
||||
_, _ = client.Write([]byte("PROXY TCP4 10.99.0.1 192.0.2.5 45678 8080\r\nGET / HTTP/1.0\r\n\r\n"))
|
||||
}()
|
||||
|
||||
got, err := bufio.NewReader(server).ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatalf("read request line: %v", err)
|
||||
}
|
||||
if got != "GET / HTTP/1.0\r\n" {
|
||||
t.Fatalf("request line = %q, want GET / HTTP/1.0", got)
|
||||
}
|
||||
if addr, ok := server.RemoteAddr().(*net.TCPAddr); !ok || addr.IP.String() != "10.99.0.1" || addr.Port != 45678 {
|
||||
t.Fatalf("RemoteAddr = %v, want 10.99.0.1:45678", server.RemoteAddr())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyProtocolAbsentKeepsRemoteAddr(t *testing.T) {
|
||||
client, server := newPipePair(t)
|
||||
defer client.Close()
|
||||
defer server.Close()
|
||||
|
||||
go func() {
|
||||
_, _ = client.Write([]byte("GET / HTTP/1.0\r\n\r\n"))
|
||||
}()
|
||||
|
||||
got, err := bufio.NewReader(server).ReadString('\n')
|
||||
if err != nil {
|
||||
t.Fatalf("read request line: %v", err)
|
||||
}
|
||||
if got != "GET / HTTP/1.0\r\n" {
|
||||
t.Fatalf("request line = %q, want GET / HTTP/1.0", got)
|
||||
}
|
||||
if server.RemoteAddr().String() != "pipe" {
|
||||
t.Fatalf("RemoteAddr = %v, want underlying conn address", server.RemoteAddr())
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyProtocolTruncatedHeaderKeepsRemoteAddr(t *testing.T) {
|
||||
client, server := newPipePair(t)
|
||||
defer server.Close()
|
||||
|
||||
go func() {
|
||||
_, _ = client.Write([]byte("PROXY TCP4 10.99.0.1"))
|
||||
_ = client.Close()
|
||||
}()
|
||||
|
||||
_, _ = server.Read(make([]byte, 64))
|
||||
if server.RemoteAddr().String() != "pipe" {
|
||||
t.Fatalf("RemoteAddr = %v, want underlying conn address", server.RemoteAddr())
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
|
||||
var (
|
||||
MAJOR_VERSION = int32(4)
|
||||
MINOR_VERSION = int32(50)
|
||||
MINOR_VERSION = int32(49)
|
||||
VERSION_NUMBER = fmt.Sprintf("%d.%02d", MAJOR_VERSION, MINOR_VERSION)
|
||||
VERSION = util.SizeLimit + " " + VERSION_NUMBER
|
||||
COMMIT = ""
|
||||
|
||||
Reference in new issue
Block a user