mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
27b34f3793 | ||
|
|
47112917ff | ||
|
|
85bed4d92e | ||
|
|
c563386675 | ||
|
|
a1c6f1fbd6 | ||
|
|
5db82e594a | ||
|
|
2b9dda7d2e | ||
|
|
d693beea87 | ||
|
|
8a6a7c63c0 | ||
|
|
32cf06d19f | ||
|
|
650f56e30c | ||
|
|
4d9274ec9d | ||
|
|
b3bcb1823e | ||
|
|
9f0e80e9f9 | ||
|
|
a667f0ad99 | ||
|
|
b40ba7c7fe | ||
|
|
310eec9a9c | ||
|
|
3834ad5c48 | ||
|
|
4ed06e9ba5 | ||
|
|
d401e374f7 | ||
|
|
4bb3911fde | ||
|
|
8cf83d3b12 | ||
|
|
d440135d70 | ||
|
|
312abfbe82 | ||
|
|
28525fb500 | ||
|
|
ba11331cc1 | ||
|
|
f95848ba7d | ||
|
|
d1e83a3b4d | ||
|
|
bf839651ec | ||
|
|
8b39bbbe2f | ||
|
|
8dfb66880e | ||
|
|
179039afaa | ||
|
|
53c709a580 | ||
|
|
85f41657dd | ||
|
|
c999851555 | ||
|
|
aaf212f7ec | ||
|
|
c6e9fd26dc | ||
|
|
6ae22b522b | ||
|
|
2bc05f70e7 | ||
|
|
30b48ad614 | ||
|
|
294337d882 | ||
|
|
612fd7206e | ||
|
|
6c82d7ddfc | ||
|
|
3df0e11ad3 | ||
|
|
117fbba5bf | ||
|
|
b2ecfad35e | ||
|
|
6acec5ed1c | ||
|
|
1a4c35d984 | ||
|
|
3469ad6b3f | ||
|
|
6175102ccf | ||
|
|
c6991dfd5f | ||
|
|
cbc24c7b24 | ||
|
|
6acb43bbbb | ||
|
|
ce9b822a8e | ||
|
|
b62066d38f | ||
|
|
deb86ff4a6 | ||
|
|
de0b969b36 | ||
|
|
1cac5d983d | ||
|
|
edee91ef0e | ||
|
|
35b5264ab7 | ||
|
|
fb450b2a5d | ||
|
|
3e8604a4c5 | ||
|
|
08a45d6c84 | ||
|
|
c2d338160e | ||
|
|
9662d71a3a | ||
|
|
9242a6cc5d | ||
|
|
49a0ee60b3 | ||
|
|
2b3e39397e | ||
|
|
3fe00996b2 | ||
|
|
082f67bfe9 | ||
|
|
0fd7222d65 | ||
|
|
7b3c63a8af | ||
|
|
4e493d596c | ||
|
|
659133fb68 | ||
|
|
530bdedf76 | ||
|
|
6ebe26a765 | ||
|
|
7540d43ee9 | ||
|
|
249c0e06ef | ||
|
|
a78c4141e5 | ||
|
|
451ec6504d | ||
|
|
f9abfd0b03 | ||
|
|
763d1946f9 | ||
|
|
0483ba3889 | ||
|
|
567d788928 | ||
|
|
b5070d6280 | ||
|
|
5fb90a3157 | ||
|
|
a16c0fabca | ||
|
|
58b46229aa | ||
|
|
c49dee4dad | ||
|
|
f7ad82e03d | ||
|
|
dffe00a822 | ||
|
|
bf13f3ced7 | ||
|
|
734178093e | ||
|
|
2a578b9033 | ||
|
|
321fda552c | ||
|
|
b02fdeabff | ||
|
|
ebd6f96d35 | ||
|
|
02cb799481 | ||
|
|
c4e6545197 | ||
|
|
358cba43ef | ||
|
|
597ab1e4bf | ||
|
|
d7c211fe8c | ||
|
|
40888b5612 | ||
|
|
abe599ee4a | ||
|
|
dd580190b4 | ||
|
|
44906f1f3b | ||
|
|
9d589b48e6 | ||
|
|
17a0f3b0e0 | ||
|
|
df4ded758e | ||
|
|
19505c1cf4 | ||
|
|
78dbac7702 | ||
|
|
faabc695b6 | ||
|
|
0530e90ea1 | ||
|
|
645ae8c57b | ||
|
|
3d07895518 | ||
|
|
a6e35e4f68 | ||
|
|
a132d44556 | ||
|
|
a345b6802d | ||
|
|
51efba43fd | ||
|
|
adabb63c18 | ||
|
|
f8b94cac0e | ||
|
|
c9177c92e5 | ||
|
|
81e57bd45e | ||
|
|
7bb58e23ea | ||
|
|
28a3a31b27 | ||
|
|
a40888d343 | ||
|
|
23f334dd13 | ||
|
|
c7e05e4e71 | ||
|
|
dff2ce5d2f | ||
|
|
1492bf7552 |
No files matched your search
@@ -38,7 +38,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@2ac3035fa4c4feed6a8272ce278b0577b93cf8e5 # v1.22
|
||||
with:
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@6c5ccdad469c9f8a2996bfecaec55a631a347034
|
||||
uses: actions/dependency-review-action@9f45b2463b475767b61721ccfef113fef513e6aa
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v2
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -12,7 +12,7 @@ jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
|
||||
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@master
|
||||
with:
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac
|
||||
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
@@ -31,7 +31,7 @@ jobs:
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.4.0
|
||||
uses: helm/chart-testing-action@v2.6.1
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
|
||||
@@ -1,14 +1,35 @@
|
||||
BINARY = weed
|
||||
|
||||
SOURCE_DIR = .
|
||||
debug ?= 0
|
||||
|
||||
all: install
|
||||
|
||||
install:
|
||||
cd weed; go install
|
||||
|
||||
warp_install:
|
||||
go install github.com/minio/warp@v0.7.6
|
||||
|
||||
full_install:
|
||||
cd weed; go install -tags "elastic gocdk sqlite ydb tikv"
|
||||
|
||||
tests:
|
||||
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
|
||||
server: install
|
||||
weed -v 4 server -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json
|
||||
|
||||
benchmark: install warp_install
|
||||
pkill weed || true
|
||||
pkill warp || true
|
||||
weed server -debug=$(debug) -s3 -filer -volume.max=0 -master.volumeSizeLimitMB=1024 -volume.preStopSeconds=1 -s3.port=8000 -s3.allowEmptyFolder=false -s3.allowDeleteBucketNotEmpty=false -s3.config=./docker/compose/s3.json &
|
||||
warp client &
|
||||
while ! nc -z localhost 8000 ; do sleep 1 ; done
|
||||
warp mixed --host=127.0.0.1:8000 --access-key=some_access_key1 --secret-key=some_secret_key1 --autoterm
|
||||
pkill warp
|
||||
pkill weed
|
||||
|
||||
# curl -o profile "http://127.0.0.1:6060/debug/pprof/profile?debug=1"
|
||||
benchmark_with_pprof: debug = 1
|
||||
benchmark_with_pprof: benchmark
|
||||
|
||||
test:
|
||||
cd weed; go test -tags "elastic gocdk sqlite ydb tikv" -v ./...
|
||||
@@ -582,6 +582,71 @@ Percentage of the requests served within a certain time (ms)
|
||||
100% 54.1 ms
|
||||
```
|
||||
|
||||
### Run WARP and launch a mixed benchmark. ###
|
||||
|
||||
```
|
||||
make benchmark
|
||||
warp: Benchmark data written to "warp-mixed-2023-10-16[102354]-l70a.csv.zst"
|
||||
Mixed operations.
|
||||
Operation: DELETE, 10%, Concurrency: 20, Ran 4m59s.
|
||||
* Throughput: 6.19 obj/s
|
||||
|
||||
Operation: GET, 45%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 279.85 MiB/s, 27.99 obj/s
|
||||
|
||||
Operation: PUT, 15%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 89.86 MiB/s, 8.99 obj/s
|
||||
|
||||
Operation: STAT, 30%, Concurrency: 20, Ran 5m0s.
|
||||
* Throughput: 18.63 obj/s
|
||||
|
||||
Cluster Total: 369.74 MiB/s, 61.79 obj/s, 0 errors over 5m0s.
|
||||
```
|
||||
|
||||
To see segmented request statistics, use the --analyze.v parameter.
|
||||
```
|
||||
warp analyze --analyze.v warp-mixed-2023-10-16[102354]-l70a.csv.zst
|
||||
18642 operations loaded... Done!
|
||||
Mixed operations.
|
||||
----------------------------------------
|
||||
Operation: DELETE - total: 1854, 10.0%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
|
||||
* Throughput: 6.19 obj/s
|
||||
|
||||
Requests considered: 1855:
|
||||
* Avg: 104ms, 50%: 30ms, 90%: 207ms, 99%: 1.355s, Fastest: 1ms, Slowest: 4.613s, StdDev: 320ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: GET - total: 8388, 45.3%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.12 +0500 +05
|
||||
* Throughput: 279.77 MiB/s, 27.98 obj/s
|
||||
|
||||
Requests considered: 8389:
|
||||
* Avg: 221ms, 50%: 106ms, 90%: 492ms, 99%: 1.739s, Fastest: 8ms, Slowest: 8.633s, StdDev: 383ms
|
||||
* TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 171ms, 99th: 669ms, Worst: 4.783s StdDev: 163ms
|
||||
* First Access: Avg: 240ms, 50%: 105ms, 90%: 511ms, 99%: 2.08s, Fastest: 12ms, Slowest: 8.633s, StdDev: 480ms
|
||||
* First Access TTFB: Avg: 88ms, Best: 2ms, 25th: 24ms, Median: 38ms, 75th: 64ms, 90th: 179ms, 99th: 919ms, Worst: 4.783s StdDev: 199ms
|
||||
* Last Access: Avg: 219ms, 50%: 106ms, 90%: 463ms, 99%: 1.782s, Fastest: 9ms, Slowest: 8.633s, StdDev: 416ms
|
||||
* Last Access TTFB: Avg: 81ms, Best: 2ms, 25th: 24ms, Median: 39ms, 75th: 65ms, 90th: 161ms, 99th: 657ms, Worst: 4.783s StdDev: 176ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: PUT - total: 2688, 14.5%, Size: 10485760 bytes. Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.115 +0500 +05
|
||||
* Throughput: 89.83 MiB/s, 8.98 obj/s
|
||||
|
||||
Requests considered: 2689:
|
||||
* Avg: 1.165s, 50%: 878ms, 90%: 2.015s, 99%: 5.74s, Fastest: 99ms, Slowest: 8.264s, StdDev: 968ms
|
||||
|
||||
----------------------------------------
|
||||
Operation: STAT - total: 5586, 30.2%, Concurrency: 20, Ran 5m0s, starting 2023-10-16 10:23:57.113 +0500 +05
|
||||
* Throughput: 18.63 obj/s
|
||||
|
||||
Requests considered: 5587:
|
||||
* Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 80ms, Fastest: 0s, Slowest: 245ms, StdDev: 17ms
|
||||
* First Access: Avg: 14ms, 50%: 10ms, 90%: 33ms, 99%: 69ms, Fastest: 0s, Slowest: 203ms, StdDev: 16ms
|
||||
* Last Access: Avg: 15ms, 50%: 11ms, 90%: 34ms, 99%: 74ms, Fastest: 0s, Slowest: 203ms, StdDev: 17ms
|
||||
|
||||
Cluster Total: 369.64 MiB/s, 61.77 obj/s, 0 errors over 5m0s.
|
||||
Total Errors:0.
|
||||
```
|
||||
|
||||
[Back to TOC](#table-of-contents)
|
||||
|
||||
## License ##
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-alpine as builder
|
||||
FROM golang:1.21-alpine as builder
|
||||
RUN apk add git g++ fuse
|
||||
RUN mkdir -p /go/src/github.com/seaweedfs/
|
||||
RUN git clone https://github.com/seaweedfs/seaweedfs /go/src/github.com/seaweedfs/seaweedfs
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-buster as builder
|
||||
FROM golang:1.21-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.20-buster as builder
|
||||
FROM golang:1.21-buster as builder
|
||||
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
+6
-6
@@ -102,12 +102,12 @@ clean:
|
||||
rm ./weed
|
||||
|
||||
certstrap:
|
||||
go get github.com/square/certstrap
|
||||
certstrap --depot-path compose/tls init --passphrase "" --common-name "SeaweedFS CA" || true
|
||||
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name volume01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name master01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name filer01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --passphrase "" --common-name client01.dev || true
|
||||
go install -v github.com/square/certstrap@latest
|
||||
certstrap --depot-path compose/tls init --curve P-256 --passphrase "" --common-name "SeaweedFS CA" || true
|
||||
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --domain localhost --common-name volume01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name master01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name filer01.dev || true
|
||||
certstrap --depot-path compose/tls request-cert --ou "SeaweedFS" --curve P-256 --passphrase "" --common-name client01.dev || true
|
||||
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" volume01.dev || true
|
||||
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" master01.dev || true
|
||||
certstrap --depot-path compose/tls sign --CA "SeaweedFS CA" filer01.dev || true
|
||||
|
||||
+12
-2
@@ -40,7 +40,10 @@
|
||||
"List",
|
||||
"Tagging",
|
||||
"Write"
|
||||
]
|
||||
],
|
||||
"account": {
|
||||
"id": "testid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "s3_tests_alt",
|
||||
@@ -101,5 +104,12 @@
|
||||
"Write"
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"accounts": [
|
||||
{
|
||||
"id" : "testid",
|
||||
"displayName": "M. Tester",
|
||||
"emailAddress": "tester@ceph.com"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -18,10 +18,10 @@ bucket prefix = yournamehere-{random}-
|
||||
|
||||
[s3 main]
|
||||
# main display_name set in vstart.sh
|
||||
display_name = s3_tests
|
||||
display_name = M. Tester
|
||||
|
||||
# main user_idname set in vstart.sh
|
||||
user_id = s3_tests
|
||||
user_id = testid
|
||||
|
||||
# main email set in vstart.sh
|
||||
email = tester@ceph.com
|
||||
|
||||
@@ -12,5 +12,9 @@ WEED_GRPC_MASTER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,clie
|
||||
WEED_GRPC_VOLUME_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
|
||||
WEED_GRPC_FILER_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
|
||||
WEED_GRPC_CLIENT_ALLOWED_COMMONNAMES="volume01.dev,master01.dev,filer01.dev,client01.dev"
|
||||
WEED_HTTPS_CLIENT_ENABLE=true
|
||||
WEED_HTTPS_VOLUME_CERT=/etc/seaweedfs/tls/volume01.dev.crt
|
||||
WEED_HTTPS_VOLUME_KEY=/etc/seaweedfs/tls/volume01.dev.key
|
||||
WEED_HTTPS_VOLUME_CA=/etc/seaweedfs/tls/SeaweedFS_CA.crt
|
||||
#GRPC_GO_LOG_SEVERITY_LEVEL=info
|
||||
#GRPC_GO_LOG_VERBOSITY_LEVEL=2
|
||||
@@ -1,22 +1,22 @@
|
||||
module github.com/seaweedfs/seaweedfs
|
||||
|
||||
go 1.20
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.110.7 // indirect
|
||||
cloud.google.com/go v0.110.8 // indirect
|
||||
cloud.google.com/go/pubsub v1.33.0
|
||||
cloud.google.com/go/storage v1.32.0
|
||||
cloud.google.com/go/storage v1.33.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.44.314
|
||||
github.com/aws/aws-sdk-go v1.45.25
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cenkalti/backoff/v4 v4.2.1
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
@@ -32,7 +32,7 @@ require (
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.8.2
|
||||
github.com/go-redsync/redsync/v4 v4.10.0
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v1.6.0
|
||||
@@ -41,8 +41,7 @@ require (
|
||||
github.com/golang/protobuf v1.5.3
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/uuid v1.3.1
|
||||
github.com/google/uuid v1.4.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
@@ -58,7 +57,7 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.16.7 // indirect
|
||||
github.com/klauspost/compress v1.17.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.8
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.9
|
||||
@@ -66,7 +65,7 @@ require (
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.11 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/mattn/go-isatty v0.0.19 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
@@ -74,26 +73,25 @@ require (
|
||||
github.com/olivere/elastic/v7 v7.0.32
|
||||
github.com/peterh/liner v1.2.2
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.1
|
||||
github.com/prometheus/client_golang v1.17.0
|
||||
github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
|
||||
github.com/prometheus/common v0.44.0 // indirect
|
||||
github.com/prometheus/procfs v0.12.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.3
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.5 // indirect
|
||||
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||
github.com/spf13/afero v1.10.0 // indirect
|
||||
github.com/spf13/cast v1.5.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.16.0
|
||||
github.com/spf13/viper v1.17.0
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.16.0
|
||||
github.com/tidwall/gjson v1.17.0
|
||||
github.com/tidwall/match v1.1.1
|
||||
github.com/tidwall/pretty v1.2.0 // indirect
|
||||
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
|
||||
@@ -110,21 +108,21 @@ require (
|
||||
go.mongodb.org/mongo-driver v1.12.1
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.34.0
|
||||
gocloud.dev/pubsub/natspubsub v0.33.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.33.0
|
||||
golang.org/x/crypto v0.13.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230321023759-10a507213a29
|
||||
golang.org/x/image v0.11.0
|
||||
golang.org/x/net v0.15.0
|
||||
golang.org/x/oauth2 v0.12.0 // indirect
|
||||
golang.org/x/sys v0.12.0
|
||||
gocloud.dev/pubsub/natspubsub v0.34.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.34.0
|
||||
golang.org/x/crypto v0.14.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230905200255-921286631fa9
|
||||
golang.org/x/image v0.12.0
|
||||
golang.org/x/net v0.17.0
|
||||
golang.org/x/oauth2 v0.13.0 // indirect
|
||||
golang.org/x/sys v0.13.0
|
||||
golang.org/x/text v0.13.0 // indirect
|
||||
golang.org/x/tools v0.11.0
|
||||
golang.org/x/tools v0.14.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.141.0
|
||||
google.golang.org/api v0.149.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect
|
||||
google.golang.org/grpc v1.58.0
|
||||
google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
|
||||
google.golang.org/grpc v1.59.0
|
||||
google.golang.org/protobuf v1.31.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
@@ -134,19 +132,19 @@ require (
|
||||
modernc.org/mathutil v1.6.0 // indirect
|
||||
modernc.org/memory v1.6.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.25.0
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/sqlite v1.26.0
|
||||
modernc.org/strutil v1.2.0
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.4.0
|
||||
github.com/arangodb/go-driver v1.6.0
|
||||
github.com/arangodb/go-driver v1.6.1
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.21.0
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.39
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.37
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.38.5
|
||||
github.com/aws/aws-sdk-go-v2 v1.22.1
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.42
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.15.1
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.40.2
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/golang-jwt/jwt/v5 v5.0.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
@@ -154,22 +152,22 @@ require (
|
||||
github.com/hashicorp/raft v1.5.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/orcaman/concurrent-map/v2 v2.0.1
|
||||
github.com/puzpuzpuz/xsync/v2 v2.5.0
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/rclone/rclone v1.63.1
|
||||
github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8
|
||||
github.com/puzpuzpuz/xsync/v2 v2.5.1
|
||||
github.com/rabbitmq/amqp091-go v1.9.0
|
||||
github.com/rclone/rclone v1.64.2
|
||||
github.com/rdleal/intervalst v1.2.0
|
||||
github.com/schollz/progressbar/v3 v3.13.1
|
||||
github.com/shirou/gopsutil/v3 v3.23.5
|
||||
github.com/shirou/gopsutil/v3 v3.23.9
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.48.8
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.53.3
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.23.0 // indirect
|
||||
cloud.google.com/go/compute v1.23.1 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v1.1.1 // indirect
|
||||
cloud.google.com/go/iam v1.1.3 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
|
||||
@@ -178,83 +176,103 @@ require (
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.6.0 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect
|
||||
github.com/ProtonMail/go-mime v0.0.0-20230322103455-7d82a3887f2f // indirect
|
||||
github.com/ProtonMail/go-srp v0.0.7 // indirect
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.7.3 // indirect
|
||||
github.com/PuerkitoBio/goquery v1.8.1 // indirect
|
||||
github.com/Unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.2 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.42 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.36 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.35 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.14 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.43 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.1.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.38 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.15.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.21.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.24.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.13.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.15.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.21.5 // indirect
|
||||
github.com/aws/smithy-go v1.14.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.17.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.25.0 // indirect
|
||||
github.com/aws/smithy-go v1.16.0 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/bradenaw/juniper v0.13.1 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
|
||||
github.com/cloudflare/circl v1.3.3 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.4.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/emersion/go-message v0.17.0 // indirect
|
||||
github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20230815062825-8fda7d206ec9 // indirect
|
||||
github.com/fatih/color v1.14.1 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/flynn/noise v1.0.0 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.8 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.10 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/go-resty/resty/v2 v2.7.0 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/s2a-go v0.1.7 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.5.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/henrybear327/Proton-API-Bridge v0.0.0-20230908065933-5bfa15b567db // indirect
|
||||
github.com/henrybear327/go-proton-api v0.0.0-20230907193451-e563407504ce // indirect
|
||||
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
|
||||
github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jlaffaye/ftp v0.2.0 // indirect
|
||||
github.com/jonboulle/clockwork v0.3.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/jtolio/noiseconn v0.0.0-20230111204749-d7ec1a08b0b8 // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.4 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.15 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.0 // indirect
|
||||
github.com/nats-io/nats.go v1.28.0 // indirect
|
||||
github.com/nats-io/nkeys v0.4.4 // indirect
|
||||
github.com/nats-io/nats.go v1.30.2 // indirect
|
||||
github.com/nats-io/nkeys v0.4.5 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.1 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.2 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.34.0 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.45.0 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
@@ -267,28 +285,30 @@ require (
|
||||
github.com/pkg/xattr v0.4.9 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/rclone/ftp v0.0.0-20230327202000-dadc1f64e87d // indirect
|
||||
github.com/rdleal/intervalst v0.0.0-20221028215511-a098aa0d2cb8 // indirect
|
||||
github.com/relvacode/iso8601 v1.3.0 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/sagikazarmark/locafero v0.3.0 // indirect
|
||||
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
|
||||
github.com/shoenig/go-m1cpu v0.1.6 // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/smartystreets/goconvey v1.8.1 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
|
||||
github.com/sourcegraph/conc v0.3.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.20-0.20230227152157-d00b379de191 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.12 // indirect
|
||||
github.com/tklauser/numcpus v0.6.1 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20231012155159-f85a672542fd // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
@@ -301,20 +321,21 @@ require (
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/mod v0.12.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/term v0.12.0 // indirect
|
||||
golang.org/x/mod v0.13.0 // indirect
|
||||
golang.org/x/sync v0.4.0 // indirect
|
||||
golang.org/x/term v0.13.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230803162519-f966b187b2e5 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230911183012-2d3300fd4832 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
|
||||
storj.io/drpc v0.0.32 // indirect
|
||||
storj.io/uplink v1.10.0 // indirect
|
||||
storj.io/common v0.0.0-20230907123639-5fd0608fd947 // indirect
|
||||
storj.io/drpc v0.0.33 // indirect
|
||||
storj.io/picobuf v0.0.2-0.20230906122608-c4ba17033c6c // indirect
|
||||
storj.io/uplink v1.12.0 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.57"
|
||||
version: 3.57.0
|
||||
appVersion: "3.59"
|
||||
version: 3.59.0
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Issuer
|
||||
metadata:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
|
||||
@@ -12,7 +12,7 @@ metadata:
|
||||
app.kubernetes.io/component: filer
|
||||
spec:
|
||||
serviceName: {{ template "seaweedfs.name" . }}-filer
|
||||
podManagementPolicy: Parallel
|
||||
podManagementPolicy: {{ .Values.filer.podManagementPolicy }}
|
||||
replicas: {{ .Values.filer.replicas }}
|
||||
{{- if (gt (int .Values.filer.updatePartition) 0) }}
|
||||
updateStrategy:
|
||||
@@ -33,6 +33,13 @@ spec:
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: filer
|
||||
{{ with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{ with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
|
||||
{{- if .Values.filer.affinity }}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
|
||||
@@ -11,7 +11,7 @@ metadata:
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
spec:
|
||||
serviceName: {{ template "seaweedfs.name" . }}-master
|
||||
podManagementPolicy: Parallel
|
||||
podManagementPolicy: {{ .Values.master.podManagementPolicy }}
|
||||
replicas: {{ .Values.master.replicas }}
|
||||
{{- if (gt (int .Values.master.updatePartition) 0) }}
|
||||
updateStrategy:
|
||||
@@ -32,6 +32,13 @@ spec:
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: master
|
||||
{{ with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{ with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
|
||||
{{- if .Values.master.affinity }}
|
||||
|
||||
@@ -24,6 +24,13 @@ spec:
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
{{ with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{ with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
|
||||
{{- if .Values.s3.tolerations }}
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
{{- if .Values.s3.ingress.enabled }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
{{- else }}
|
||||
apiVersion: extensions/v1beta1
|
||||
{{- end }}
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: ingress-{{ template "seaweedfs.name" . }}-s3
|
||||
namespace: {{ .Release.Namespace }}
|
||||
{{- if .Values.s3.ingress.annotations }}
|
||||
annotations:
|
||||
{{ tpl .Values.s3.ingress.annotations . | nindent 4 | trim }}
|
||||
{{- end }}
|
||||
labels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: s3
|
||||
spec:
|
||||
ingressClassName: {{ .Values.s3.ingress.className | quote }}
|
||||
tls:
|
||||
{{ .Values.s3.ingress.tls | default list | toYaml | nindent 6}}
|
||||
rules:
|
||||
- http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: ImplementationSpecific
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ template "seaweedfs.name" . }}-s3
|
||||
port:
|
||||
number: {{ .Values.s3.port }}
|
||||
#name:
|
||||
{{- else }}
|
||||
serviceName: {{ template "seaweedfs.name" . }}-s3
|
||||
servicePort: {{ .Values.s3.port }}
|
||||
{{- end }}
|
||||
{{- if .Values.s3.ingress.host }}
|
||||
host: {{ .Values.s3.ingress.host }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -9,6 +9,7 @@ metadata:
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
spec:
|
||||
internalTrafficPolicy: {{ .Values.s3.internalTrafficPolicy | default "Cluster" }}
|
||||
ports:
|
||||
- name: "swfs-s3"
|
||||
port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
|
||||
|
||||
@@ -13,10 +13,38 @@ data:
|
||||
security.toml: |-
|
||||
# this file is read by master, volume server, and filer
|
||||
|
||||
{{- if .Values.global.securityConfig.jwtSigning.volumeWrite }}
|
||||
# the jwt signing key is read by master and volume server
|
||||
# a jwt expires in 10 seconds
|
||||
[jwt.signing]
|
||||
key = "{{ randAlphaNum 10 | b64enc }}"
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.securityConfig.jwtSigning.volumeRead }}
|
||||
# this jwt signing key is read by master and volume server, and it is used for read operations:
|
||||
# - the Master server generates the JWT, which can be used to read a certain file on a volume server
|
||||
# - the Volume server validates the JWT on reading
|
||||
[jwt.signing.read]
|
||||
key = "{{ randAlphaNum 10 | b64enc }}"
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.securityConfig.jwtSigning.filerWrite }}
|
||||
# If this JWT key is configured, Filer only accepts writes over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing]
|
||||
key = "{{ randAlphaNum 10 | b64enc }}"
|
||||
{{- end }}
|
||||
|
||||
{{- if .Values.global.securityConfig.jwtSigning.filerRead }}
|
||||
# If this JWT key is configured, Filer only accepts reads over HTTP if they are signed with this JWT:
|
||||
# - f.e. the S3 API Shim generates the JWT
|
||||
# - the Filer server validates the JWT on writing
|
||||
# the jwt defaults to expire after 10 seconds.
|
||||
[jwt.filer_signing.read]
|
||||
key = "{{ randAlphaNum 10 | b64enc }}"
|
||||
{{- end }}
|
||||
|
||||
# all grpc tls authentications are mutual
|
||||
# the values for the following ca, cert, and key are paths to the PERM files.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if and .Values.global.enableSecurity (not .Values.certificates.externalCertificates.enabled)}}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
|
||||
@@ -10,6 +10,7 @@ metadata:
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
spec:
|
||||
clusterIP: None
|
||||
internalTrafficPolicy: {{ .Values.volume.internalTrafficPolicy | default "Cluster" }}
|
||||
ports:
|
||||
- name: "swfs-volume"
|
||||
port: {{ .Values.volume.port }}
|
||||
@@ -27,4 +28,4 @@ spec:
|
||||
{{- end }}
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
app.kubernetes.io/component: volume
|
||||
app.kubernetes.io/component: volume
|
||||
@@ -12,7 +12,7 @@ metadata:
|
||||
spec:
|
||||
serviceName: {{ template "seaweedfs.name" . }}-volume
|
||||
replicas: {{ .Values.volume.replicas }}
|
||||
podManagementPolicy: Parallel
|
||||
podManagementPolicy: {{ .Values.volume.podManagementPolicy }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ template "seaweedfs.name" . }}
|
||||
@@ -26,6 +26,13 @@ spec:
|
||||
helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: volume
|
||||
{{ with .Values.podLabels }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
annotations:
|
||||
{{ with .Values.podAnnotations }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if .Values.volume.affinity }}
|
||||
affinity:
|
||||
|
||||
@@ -10,6 +10,12 @@ global:
|
||||
restartPolicy: Always
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
securityConfig:
|
||||
jwtSigning:
|
||||
volumeWrite: true
|
||||
volumeRead: false
|
||||
filerWrite: false
|
||||
filerRead: false
|
||||
certificates:
|
||||
alphacrds: false
|
||||
monitoring:
|
||||
@@ -88,6 +94,9 @@ master:
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
## Set podManagementPolicy
|
||||
podManagementPolicy: Parallel
|
||||
|
||||
# Resource requests, limits, etc. for the master cluster placement. This
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
@@ -263,6 +272,9 @@ volume:
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
## Set podManagementPolicy
|
||||
podManagementPolicy: Parallel
|
||||
|
||||
# Affinity Settings
|
||||
# Commenting out or setting as empty the affinity variable, will allow
|
||||
# deployment to single node services such as Minikube
|
||||
@@ -396,6 +408,9 @@ filer:
|
||||
extraVolumes: ""
|
||||
extraVolumeMounts: ""
|
||||
|
||||
## Set podManagementPolicy
|
||||
podManagementPolicy: Parallel
|
||||
|
||||
# Affinity Settings
|
||||
# Commenting out or setting as empty the affinity variable, will allow
|
||||
# deployment to single node services such as Minikube
|
||||
@@ -621,6 +636,12 @@ s3:
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 10
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
# host: false for "*" hostname
|
||||
host: "seaweedfs.cluster.local"
|
||||
|
||||
certificates:
|
||||
commonName: "SeaweedFS CA"
|
||||
ipAddresses: []
|
||||
@@ -628,3 +649,13 @@ certificates:
|
||||
keySize: 2048
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
externalCertificates:
|
||||
# This will avoid the need to use cert-manager and will rely on providing your own external certificates and CA
|
||||
# you will need to store your provided certificates in the secret read by the different services:
|
||||
# seaweedfs-master-cert, seaweedfs-filer-cert, etc. Can see any statefulset definition to see secret names
|
||||
enabled: false
|
||||
|
||||
# Labels to be added to all the created pods
|
||||
podLabels: {}
|
||||
# Annotations to be added to all the created pods
|
||||
podAnnotations: {}
|
||||
@@ -13,12 +13,12 @@ public class FilerClient extends FilerGrpcClient {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
|
||||
|
||||
public FilerClient(String host, int grpcPort) {
|
||||
super(host, grpcPort-10000, grpcPort);
|
||||
public FilerClient(String filerHost, int filerGrpcPort) {
|
||||
super(filerHost, filerGrpcPort-10000, filerGrpcPort);
|
||||
}
|
||||
|
||||
public FilerClient(String host, int port, int grpcPort) {
|
||||
super(host, port, grpcPort);
|
||||
public FilerClient(String filerHost, int filerPort, int filerGrpcPort) {
|
||||
super(filerHost, filerPort, filerGrpcPort);
|
||||
}
|
||||
|
||||
public static String toFileId(FilerProto.FileId fid) {
|
||||
|
||||
@@ -424,6 +424,7 @@ message FilerConf {
|
||||
string data_center = 9;
|
||||
string rack = 10;
|
||||
string data_node = 11;
|
||||
uint32 max_file_name_length = 12;
|
||||
}
|
||||
repeated PathConf locations = 2;
|
||||
}
|
||||
|
||||
@@ -1,2 +1,8 @@
|
||||
# Test run scratch data
|
||||
/s3-tests
|
||||
/tmp
|
||||
|
||||
# Test run outputs
|
||||
weed.log
|
||||
compat.raw.txt
|
||||
compat.summary.txt
|
||||
@@ -1,11 +1,21 @@
|
||||
# the tests only support python 3.6, not newer
|
||||
FROM ubuntu:latest
|
||||
#FROM ubuntu:latest
|
||||
FROM python:3.6.15-slim-buster
|
||||
|
||||
RUN apt-get update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
|
||||
# Installed required system deps
|
||||
RUN apt-get update \
|
||||
&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get install -y git-core sudo tzdata
|
||||
|
||||
# Install python deps
|
||||
RUN pip install virtualenv
|
||||
|
||||
# Clone Ceph S3 tests
|
||||
RUN git clone https://github.com/ceph/s3-tests.git
|
||||
|
||||
WORKDIR s3-tests
|
||||
|
||||
# we pin a certain commit
|
||||
# Pin to a certain commit on ceph/s3-tests
|
||||
# https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
|
||||
RUN git checkout 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e
|
||||
|
||||
RUN ./bootstrap
|
||||
@@ -0,0 +1,9 @@
|
||||
.PHONY: all build-image test
|
||||
|
||||
all: build-image test
|
||||
|
||||
build-image:
|
||||
./prepare.sh
|
||||
|
||||
test:
|
||||
./run.sh
|
||||
@@ -1,13 +1,245 @@
|
||||
# Running S3 Compatibility tests against SeaweedFS
|
||||
|
||||
This is using [the tests from CephFS](https://github.com/ceph/s3-tests).
|
||||
This is using [the tests from CephFS][s3-tests].
|
||||
|
||||
[s3-tests]: https://github.com/ceph/s3-tests
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- have Docker installed
|
||||
- have [Docker][docker] installed
|
||||
- this has been executed on Mac. On Linux, the hostname in `s3tests.conf` needs to be adjusted.
|
||||
|
||||
[docker]: https://docs.docker.com
|
||||
|
||||
## Running tests
|
||||
|
||||
- `./prepare.sh` to build the docker image
|
||||
- `./run.sh` to execute all tests
|
||||
To build the docker image that is used for the tests:
|
||||
|
||||
```console
|
||||
./prepare.sh
|
||||
```
|
||||
|
||||
To execute all tests:
|
||||
|
||||
```console
|
||||
./run.sh
|
||||
```
|
||||
To see debug output including all commands run by the script:
|
||||
|
||||
```console
|
||||
DEBUG=y ./run.sh
|
||||
```
|
||||
|
||||
> [!WARNING]
|
||||
>
|
||||
> If your output does *not* look like the content in [`results.summary.txt`](./results.summary.txt)
|
||||
> and it is full of HTTP level exceptions, there is likely an error contacting the `weed` server from
|
||||
> the container that is runnin the S3 compatibility tests.
|
||||
>
|
||||
> There are at least a couple ways to solve this:
|
||||
>
|
||||
> - Modify your `docker` setup to ensure `host.docker.internal` is connected to your host running `weed`
|
||||
> - Use `--net=host` and modify `host` in `s3tests.conf` to `localhost`
|
||||
>
|
||||
> The `--net=host` solution is potentially *unsafe*, as the container running [s3-tests][s3-tests] could
|
||||
> visit unexpected websites or use the host-passthrough internet access maliciously.
|
||||
>
|
||||
> If you are OK with the risk of allowing `--net=host`:
|
||||
>
|
||||
> - Set `host = localhost` in `s3tests.conf`
|
||||
> - Set `DOCKER_NET_HOST=y` when running `run.sh`
|
||||
|
||||
## Most recent results
|
||||
|
||||
See [`results.summary.txt`](./results.summary.txt) for the latest results of compatibility testing (with the caveat that `s3-tests` is pinned to [`ceph/s3-tests` @ 9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e](https://github.com/ceph/s3-tests/commit/9a6a1e9f197fc9fb031b809d1e057635c2ff8d4e)).
|
||||
|
||||
The file is reproduced below for ease of access:
|
||||
|
||||
```
|
||||
/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
|
||||
warnings.warn(warning, PythonDeprecationWarning)
|
||||
s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_file ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
|
||||
s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
|
||||
s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
|
||||
s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
|
||||
ERROR
|
||||
|
||||
======================================================================
|
||||
ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
|
||||
self.test(*self.arg)
|
||||
File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
|
||||
'DisplayName': acl_response['Owner']['DisplayName'],
|
||||
KeyError: 'Owner'
|
||||
-------------------- >> begin captured logging << --------------------
|
||||
```
|
||||
@@ -0,0 +1,186 @@
|
||||
/s3-tests/virtualenv/lib/python3.6/site-packages/boto3/compat.py:88: PythonDeprecationWarning: Boto3 will no longer support Python 3.6 starting May 30, 2022. To continue receiving service updates, bug fixes, and security updates please upgrade to Python 3.7 or later. More information can be found here: https://aws.amazon.com/blogs/developer/python-support-policy-updates-for-aws-sdks-and-tools/
|
||||
warnings.warn(warning, PythonDeprecationWarning)
|
||||
s3tests_boto3.functional.test_s3.test_bucket_list_return_data ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_write_to_nonexist_bucket ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_read_not_exist ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_requestid_matches_header_on_error ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multi_object_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_object_delete_key_limit ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multi_objectv2_delete_key_limit ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_head_zero_bytes ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_check_etag ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_cache_control ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_expires ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_read_update_read_delete ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_metadata_replaced_on_put ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_write_file ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_anonymous_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_no_content_type ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_authenticated_request_bad_access_key ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_success_code ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_invalid_success_code ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_larger_than_chunk ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_set_key_from_filename ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_ignored_header ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_case_insensitive_condition_fields ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_escaped_field_values ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_success_redirect_action ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_signature ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_access_key ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_date_format ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_no_key_specified ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_signature ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_policy_condition ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_user_specified_header ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_request_missing_policy_specified_field ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_condition_is_case_sensitive ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_expires_is_case_sensitive ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_expired_policy ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_request_field_value ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_expires_condition ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_conditions_list ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_size_limit_exceeded ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_missing_content_length_argument ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_invalid_content_length_argument ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_upload_size_below_minimum ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_empty_conditions ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmatch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifnonematch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifmodifiedsince_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_object_ifunmodifiedsince_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_overwrite_existed_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifmatch_nonexisted_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_nonexisted_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_object_ifnonmatch_overwrite_existed_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_delete_key_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_object_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_response_headers ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_bucket_gone ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_authenticated_object_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_not_expired ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_range_zero ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_max_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_get_x_amz_expires_out_positive_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_anon_put ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_anon_put_write_access ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_put_authenticated ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_raw_put_authenticated_expired ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_canned_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_write ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_writeacp ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_read ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_acl_readacp ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_header_acl_grants ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_private ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_private ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicread ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicread ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_object_publicreadwrite ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_private_objectv2_publicreadwrite ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_private ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicread ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicread_object_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_private ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicread ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_access_bucket_publicreadwrite_object_publicreadwrite ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_bucket_create_special_key_names ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_zero_size ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_same_bucket ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_verify_contenttype ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_to_itself ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_to_itself_with_metadata ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_diff_bucket ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_bucket ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_not_owned_object_bucket ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_canned_acl ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_retaining_metadata ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_replacing_metadata ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_bucket_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_copy_key_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_empty ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_small ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_small ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_invalid_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_improper_range ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_without_range ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_copy_special_names ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_resend_part ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_size_too_small ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_contents ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_overwrite_existing_object ... ok
|
||||
s3tests_boto3.functional.test_s3.test_abort_multipart_upload ... ok
|
||||
s3tests_boto3.functional.test_s3.test_abort_multipart_upload_not_found ... ok
|
||||
s3tests_boto3.functional.test_s3.test_list_multipart_upload ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_missing_part ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_upload_incorrect_etag ... ok
|
||||
s3tests_boto3.functional.test_s3.test_100_continue ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_read_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_4mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_write_8mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_conditional_write_1mb ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_dual_conditional_write_1mb ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_atomic_write_bucket_gone ... ok
|
||||
s3tests_boto3.functional.test_s3.test_atomic_multipart_upload_write ... ok
|
||||
s3tests_boto3.functional.test_s3.test_multipart_resend_first_finishes_last ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_big_request_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_skip_leading_bytes_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_return_trailing_bytes_response_code ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_invalid_range ... ok
|
||||
s3tests_boto3.functional.test_s3.test_ranged_request_empty_object ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_obj_tagging ... ok
|
||||
s3tests_boto3.functional.test_s3.test_get_obj_head_tagging ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_max_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_max_kvsize_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_key_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_excess_val_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_modify_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_put_delete_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_post_object_tags_anonymous_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_post_object_tags_authenticated_request ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_put_obj_with_tags ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_multi_delete_object_with_retention ... ERROR
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_with_bypass ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_governance_without_bypass ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_object_lock_changing_mode_from_compliance ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_good ... ok
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifmatch_failed ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_good ... FAIL
|
||||
s3tests_boto3.functional.test_s3.test_copy_object_ifnonematch_failed ... ok
|
||||
s3tests_boto3.functional.test_s3.test_object_read_unreadable ... FAIL
|
||||
ERROR
|
||||
|
||||
======================================================================
|
||||
ERROR: s3tests_boto3.functional.test_s3.test_bucket_list_return_data
|
||||
----------------------------------------------------------------------
|
||||
Traceback (most recent call last):
|
||||
File "/s3-tests/virtualenv/lib/python3.6/site-packages/nose/case.py", line 198, in runTest
|
||||
self.test(*self.arg)
|
||||
File "/s3-tests/s3tests_boto3/functional/test_s3.py", line 1669, in test_bucket_list_return_data
|
||||
'DisplayName': acl_response['Owner']['DisplayName'],
|
||||
KeyError: 'Owner'
|
||||
-------------------- >> begin captured logging << --------------------
|
||||
@@ -1,24 +1,88 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -ex
|
||||
CONTAINER_NAME=${CONTAINER_NAME:-s3test-instance}
|
||||
CONF_FILE=${CONF_FILE:-s3tests.conf}
|
||||
WEED_BIN=${WEED_BIN:-../../../weed/weed}
|
||||
TEST_RAW_OUTPUT_FILE=${TEST_RAW_OUTPUT_FILE:-compat.raw.txt}
|
||||
TEST_PROCESSED_OUTPUT_FILE=${TEST_PROCESSED_OUTPUT_FILE:-compat.summary.txt}
|
||||
|
||||
# Set up debugging for this bash script if DEBUG is set
|
||||
if [ -n "${DEBUG}" ]; then
|
||||
echo -e "DEBUG set [${DEBUG}], enabling debugging output...";
|
||||
set -ex
|
||||
fi
|
||||
|
||||
# Reset from possible previous test run
|
||||
killall -9 weed || echo "already stopped"
|
||||
rm -Rf tmp
|
||||
mkdir tmp
|
||||
docker stop s3test-instance || echo "already stopped"
|
||||
docker stop $CONTAINER_NAME || echo "already stopped"
|
||||
|
||||
# Ensure ulimit is set to reasonable value
|
||||
ulimit -n 10000
|
||||
../../../weed/weed server -filer -s3 -volume.max 0 -master.volumeSizeLimitMB 5 -dir "$(pwd)/tmp" 1>&2>weed.log &
|
||||
|
||||
# Start weed w/ filer + s3 in the background
|
||||
$WEED_BIN server \
|
||||
-filer \
|
||||
-s3 \
|
||||
-volume.max 0 \
|
||||
-master.volumeSizeLimitMB 5 \
|
||||
-dir "$(pwd)/tmp" \
|
||||
1>&2>weed.log &
|
||||
|
||||
# Wait for master to start up
|
||||
echo -e "\n[info] waiting for master @ 9333...";
|
||||
until curl --output /dev/null --silent --head --fail http://127.0.0.1:9333; do
|
||||
printf '.'
|
||||
sleep 5
|
||||
printf '.';
|
||||
sleep 5;
|
||||
done
|
||||
sleep 3
|
||||
sleep 3;
|
||||
|
||||
rm -Rf logs-full.txt logs-summary.txt
|
||||
# docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v `pwd`/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py:test_get_obj_tagging -v -a 'resource=object,!bucket-policy,!versioning,!encryption'
|
||||
docker run --name s3test-instance --rm -e S3TEST_CONF=s3tests.conf -v "$(pwd)"/s3tests.conf:/s3-tests/s3tests.conf -it s3tests ./virtualenv/bin/nosetests s3tests_boto3/functional/test_s3.py -v -a 'resource=object,!bucket-policy,!versioning,!encryption' | sed -n -e '/botocore.hooks/!p;//q' | tee logs-summary.txt
|
||||
# Wait for s3 to start up
|
||||
echo -e "\n[info] waiting for S3 @ 8333...";
|
||||
until curl --output /dev/null --silent --fail http://127.0.0.1:8333; do
|
||||
printf '.';
|
||||
sleep 5;
|
||||
done
|
||||
sleep 3;
|
||||
|
||||
docker stop s3test-instance || echo "already stopped"
|
||||
killall -9 weed
|
||||
# Determine whether docker net
|
||||
DOCKER_NET_HOST_ARGS=""
|
||||
if [ -n "${DOCKER_NET_HOST}" ]; then
|
||||
DOCKER_NET_HOST_ARGS="--net=host"
|
||||
echo -e "\n[info] setting docker to het nost"
|
||||
fi
|
||||
|
||||
echo -e "\n[warn] You may have to run with UNFILTERED=y to disable output filtering, if you get the broken pipe error";
|
||||
echo -e "\n[info] running tests with unfiltered output...";
|
||||
docker run \
|
||||
--name $CONTAINER_NAME \
|
||||
--rm \
|
||||
${DOCKER_NET_HOST_ARGS} \
|
||||
-e S3TEST_CONF=$CONF_FILE \
|
||||
-v "$(pwd)"/$CONF_FILE:/s3-tests/s3tests.conf \
|
||||
-it \
|
||||
s3tests \
|
||||
./virtualenv/bin/nosetests \
|
||||
s3tests_boto3/functional/test_s3.py \
|
||||
-v \
|
||||
-a 'resource=object,!bucket-policy,!versioning,!encryption' \
|
||||
| tee ${TEST_RAW_OUTPUT_FILE}
|
||||
|
||||
# If the summary logs are present, process them
|
||||
if [ -f "${TEST_RAW_OUTPUT_FILE}" ]; then
|
||||
cat ${TEST_RAW_OUTPUT_FILE} | sed -n -e '/botocore.hooks/!p;//q' | tee ${TEST_PROCESSED_OUTPUT_FILE}
|
||||
echo -e "\n[info] ✅ Successfully wrote processed output @ [${TEST_PROCESSED_OUTPUT_FILE}]";
|
||||
if [ -z "${TEST_KEEP_RAW_OUTPUT}" ]; then
|
||||
echo -e "\n[info] removing test raw output file @ [${TEST_RAW_OUTPUT_FILE}] (to disable this, set TEST_KEEP_RAW_OUTPUT=y)...";
|
||||
rm -rf ${TEST_RAW_OUTPUT_FILE};
|
||||
fi
|
||||
else
|
||||
echo -e "\n[warn] failed to find raw output @ [${TEST_RAW_OUTPUT_FILE}]";
|
||||
fi
|
||||
|
||||
echo -e "\n[info] stopping [${CONTAINER_NAME}] container...";
|
||||
docker stop $CONTAINER_NAME || echo "[info] already stopped";
|
||||
|
||||
echo -e "\n[info] stopping seaweedfs processes (all, via kill -9)...";
|
||||
killall -9 weed;
|
||||
@@ -4,6 +4,14 @@
|
||||
# host set for rgw in vstart.sh
|
||||
host = host.docker.internal
|
||||
|
||||
## NOTE: if running docker --net=host (e.x. `DOCKER_NET_HOST=y ./run.sh`)
|
||||
## then use 'localhost', so the container can more easily access your *local* weed instance
|
||||
##
|
||||
## This is *unsafe* -- it is possible that this script will do malicious
|
||||
## things with host-level access. only run this if you trust the programmatic content of
|
||||
## ceph/s3-tests (https://github.com/ceph/s3-tests) which will be using this file
|
||||
#host = localhost
|
||||
|
||||
# port set for rgw in vstart.sh
|
||||
port = 8333
|
||||
|
||||
|
||||
@@ -33,8 +33,8 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
|
||||
|
||||
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
|
||||
t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
|
||||
glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
|
||||
*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
|
||||
glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v name %s",
|
||||
*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t, n.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ func (lc *LockClient) StartLock(key string, owner string) (lock *LiveLock) {
|
||||
lc: lc,
|
||||
}
|
||||
go func() {
|
||||
util.RetryForever("create lock:"+key, func() error {
|
||||
util.RetryUntil("create lock:"+key, func() error {
|
||||
errorMessage, err := lock.doLock(lock_manager.MaxDuration)
|
||||
if err != nil {
|
||||
glog.Infof("create lock %s: %s", key, err)
|
||||
@@ -98,7 +98,7 @@ func (lc *LockClient) doNewLock(key string, lockDuration time.Duration, owner st
|
||||
lockDuration = lc.maxLockDuration
|
||||
needRenewal = true
|
||||
}
|
||||
util.RetryForever("create lock:"+key, func() error {
|
||||
util.RetryUntil("create lock:"+key, func() error {
|
||||
errorMessage, err := lock.doLock(lockDuration)
|
||||
if err != nil {
|
||||
time.Sleep(time.Second)
|
||||
@@ -148,7 +148,7 @@ func (lc *LockClient) keepLock(lock *LiveLock) {
|
||||
select {
|
||||
case <-ticker:
|
||||
// renew the lock if lock.expireAtNs is still greater than now
|
||||
util.RetryForever("keep lock:"+lock.key, func() error {
|
||||
util.RetryUntil("keep lock:"+lock.key, func() error {
|
||||
lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
|
||||
if lockDuration > lc.maxLockDuration {
|
||||
lockDuration = lc.maxLockDuration
|
||||
|
||||
@@ -169,7 +169,7 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
|
||||
go stats_collect.StartMetricsServer(*f.bindIp, *f.metricsHttpPort)
|
||||
|
||||
filerAddress := util.JoinHostPort(*f.ip, *f.port)
|
||||
filerAddress := pb.NewServerAddress(*f.ip, *f.port, *f.portGrpc).String()
|
||||
startDelay := time.Duration(2)
|
||||
if *filerStartS3 {
|
||||
filerS3Options.filer = &filerAddress
|
||||
|
||||
@@ -111,7 +111,7 @@ func runFilerRemoteGateway(cmd *Command, args []string) bool {
|
||||
|
||||
// synchronize /buckets folder
|
||||
fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
|
||||
util.RetryForever("filer.remote.sync buckets", func() error {
|
||||
util.RetryUntil("filer.remote.sync buckets", func() error {
|
||||
return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
|
||||
}, func(err error) bool {
|
||||
if err != nil {
|
||||
|
||||
@@ -90,7 +90,7 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
|
||||
|
||||
if dir != "" {
|
||||
fmt.Printf("synchronize %s to remote storage...\n", dir)
|
||||
util.RetryForever("filer.remote.sync "+dir, func() error {
|
||||
util.RetryUntil("filer.remote.sync "+dir, func() error {
|
||||
return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
|
||||
}, func(err error) bool {
|
||||
if err != nil {
|
||||
|
||||
@@ -99,7 +99,7 @@ func runFilerReplicate(cmd *Command, args []string) bool {
|
||||
if m.OldEntry != nil && m.NewEntry == nil {
|
||||
glog.V(1).Infof("delete: %s", key)
|
||||
} else if m.OldEntry == nil && m.NewEntry != nil {
|
||||
glog.V(1).Infof(" add: %s", key)
|
||||
glog.V(1).Infof("add: %s", key)
|
||||
} else {
|
||||
glog.V(1).Infof("modify: %s", key)
|
||||
}
|
||||
|
||||
@@ -288,8 +288,8 @@ func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers strin
|
||||
}
|
||||
|
||||
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
|
||||
slices.SortFunc(peers, func(a, b pb.ServerAddress) bool {
|
||||
return strings.Compare(string(a), string(b)) < 0
|
||||
slices.SortFunc(peers, func(a, b pb.ServerAddress) int {
|
||||
return strings.Compare(string(a), string(b))
|
||||
})
|
||||
if len(peers) <= 0 {
|
||||
return true
|
||||
|
||||
+27
-1
@@ -3,7 +3,9 @@ package command
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -42,6 +44,8 @@ type S3Options struct {
|
||||
domainName *string
|
||||
tlsPrivateKey *string
|
||||
tlsCertificate *string
|
||||
tlsCACertificate *string
|
||||
tlsVerifyClientCert *bool
|
||||
metricsHttpPort *int
|
||||
allowEmptyFolder *bool
|
||||
allowDeleteBucketNotEmpty *bool
|
||||
@@ -65,6 +69,8 @@ func init() {
|
||||
s3StandaloneOptions.auditLogConfig = cmdS3.Flag.String("auditLogConfig", "", "path to the audit log config file")
|
||||
s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
|
||||
s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
|
||||
s3StandaloneOptions.tlsCACertificate = cmdS3.Flag.String("cacert.file", "", "path to the TLS CA certificate file")
|
||||
s3StandaloneOptions.tlsVerifyClientCert = cmdS3.Flag.Bool("tlsVerifyClientCert", false, "whether to verify the client's certificate")
|
||||
s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
|
||||
s3StandaloneOptions.allowDeleteBucketNotEmpty = cmdS3.Flag.Bool("allowDeleteBucketNotEmpty", true, "allow recursive deleting all entries along with bucket")
|
||||
@@ -289,7 +295,27 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
|
||||
glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
|
||||
}
|
||||
httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
|
||||
|
||||
caCertPool := x509.NewCertPool()
|
||||
if *s3Options.tlsCACertificate != "" {
|
||||
// load CA certificate file and add it to list of client CAs
|
||||
caCertFile, err := ioutil.ReadFile(*s3opt.tlsCACertificate)
|
||||
if err != nil {
|
||||
glog.Fatalf("error reading CA certificate: %v", err)
|
||||
}
|
||||
caCertPool.AppendCertsFromPEM(caCertFile)
|
||||
}
|
||||
|
||||
clientAuth := tls.NoClientCert
|
||||
if *s3Options.tlsVerifyClientCert {
|
||||
clientAuth = tls.RequireAndVerifyClientCert
|
||||
}
|
||||
|
||||
httpS.TLSConfig = &tls.Config{
|
||||
GetCertificate: s3opt.GetCertificateWithUpdate,
|
||||
ClientAuth: clientAuth,
|
||||
ClientCAs: caCertPool,
|
||||
}
|
||||
if *s3opt.portHttps == 0 {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
|
||||
if s3ApiLocalListener != nil {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
recursive_delete = false
|
||||
#max_file_name_length = 255
|
||||
|
||||
####################################################
|
||||
# The following are filer store options
|
||||
|
||||
@@ -144,6 +144,8 @@ func init() {
|
||||
s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
||||
s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
|
||||
s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
|
||||
s3Options.tlsCACertificate = cmdServer.Flag.String("s3.cacert.file", "", "path to the TLS CA certificate file")
|
||||
s3Options.tlsVerifyClientCert = cmdServer.Flag.Bool("s3.tlsVerifyClientCert", false, "whether to verify the client's certificate")
|
||||
s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
|
||||
s3Options.auditLogConfig = cmdServer.Flag.String("s3.auditLogConfig", "", "path to the audit log config file")
|
||||
s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/security"
|
||||
"github.com/seaweedfs/seaweedfs/weed/server/constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
|
||||
|
||||
"google.golang.org/grpc/reflection"
|
||||
@@ -241,7 +242,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
|
||||
*v.idxFolder,
|
||||
volumeNeedleMapKind,
|
||||
v.masters, 5, *v.dataCenter, *v.rack,
|
||||
v.masters, constants.VolumePulseSeconds, *v.dataCenter, *v.rack,
|
||||
v.whiteList,
|
||||
*v.fixJpgOrientation, *v.readMode,
|
||||
*v.compactionMBPerSecond,
|
||||
|
||||
@@ -73,11 +73,11 @@ func TestCompactFileChunksRealCase(t *testing.T) {
|
||||
}
|
||||
|
||||
func printChunks(name string, chunks []*filer_pb.FileChunk) {
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) bool {
|
||||
slices.SortFunc(chunks, func(a, b *filer_pb.FileChunk) int {
|
||||
if a.Offset == b.Offset {
|
||||
return a.ModifiedTsNs < b.ModifiedTsNs
|
||||
return int(a.ModifiedTsNs - b.ModifiedTsNs)
|
||||
}
|
||||
return a.Offset < b.Offset
|
||||
return int(a.Offset - b.Offset)
|
||||
})
|
||||
for _, chunk := range chunks {
|
||||
glog.V(0).Infof("%s chunk %s [%10d,%10d)", name, chunk.GetFileIdString(), chunk.Offset, chunk.Offset+int64(chunk.Size))
|
||||
|
||||
@@ -30,14 +30,20 @@ func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOff
|
||||
isStart: false,
|
||||
})
|
||||
}
|
||||
slices.SortFunc(points, func(a, b *Point) bool {
|
||||
slices.SortFunc(points, func(a, b *Point) int {
|
||||
if a.x != b.x {
|
||||
return a.x < b.x
|
||||
return int(a.x - b.x)
|
||||
}
|
||||
if a.ts != b.ts {
|
||||
return a.ts < b.ts
|
||||
return int(a.ts - b.ts)
|
||||
}
|
||||
return !a.isStart
|
||||
if a.isStart {
|
||||
return -1
|
||||
}
|
||||
if b.isStart {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
})
|
||||
|
||||
var prevX int64
|
||||
|
||||
+8
-2
@@ -50,9 +50,10 @@ type Filer struct {
|
||||
FilerConf *FilerConf
|
||||
RemoteStorage *FilerRemoteStorage
|
||||
Dlm *lock_manager.DistributedLockManager
|
||||
MaxFilenameLength uint32
|
||||
}
|
||||
|
||||
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
|
||||
func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress, filerGroup string, collection string, replication string, dataCenter string, maxFilenameLength uint32, notifyFn func()) *Filer {
|
||||
f := &Filer{
|
||||
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
|
||||
fileIdDeletionQueue: util.NewUnboundedQueue(),
|
||||
@@ -61,6 +62,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
|
||||
RemoteStorage: NewFilerRemoteStorage(),
|
||||
UniqueFilerId: util.RandomInt32(),
|
||||
Dlm: lock_manager.NewDistributedLockManager(filerHost),
|
||||
MaxFilenameLength: maxFilenameLength,
|
||||
}
|
||||
if f.UniqueFilerId < 0 {
|
||||
f.UniqueFilerId = -f.UniqueFilerId
|
||||
@@ -194,12 +196,16 @@ func (f *Filer) RollbackTransaction(ctx context.Context) error {
|
||||
return f.Store.RollbackTransaction(ctx)
|
||||
}
|
||||
|
||||
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool) error {
|
||||
func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFromOtherCluster bool, signatures []int32, skipCreateParentDir bool, maxFilenameLength uint32) error {
|
||||
|
||||
if string(entry.FullPath) == "/" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if entry.FullPath.IsLongerFileName(maxFilenameLength) {
|
||||
return fmt.Errorf("entry name too long")
|
||||
}
|
||||
|
||||
oldEntry, _ := f.FindEntry(ctx, entry.FullPath)
|
||||
|
||||
/*
|
||||
|
||||
@@ -166,6 +166,9 @@ func mergePathConf(a, b *filer_pb.FilerConf_PathConf) {
|
||||
a.VolumeGrowthCount = b.VolumeGrowthCount
|
||||
}
|
||||
a.ReadOnly = b.ReadOnly || a.ReadOnly
|
||||
if b.MaxFileNameLength > 0 {
|
||||
a.MaxFileNameLength = b.MaxFileNameLength
|
||||
}
|
||||
a.DataCenter = util.Nvl(b.DataCenter, a.DataCenter)
|
||||
a.Rack = util.Nvl(b.Rack, a.Rack)
|
||||
a.DataNode = util.Nvl(b.DataNode, a.DataNode)
|
||||
|
||||
@@ -43,7 +43,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
|
||||
entry.Chunks = append(entry.GetChunks(), uploadResult.ToPbFileChunk(assignResult.Fid, offset, time.Now().UnixNano()))
|
||||
|
||||
// update the entry
|
||||
err = f.CreateEntry(context.Background(), entry, false, false, nil, false)
|
||||
err = f.CreateEntry(context.Background(), entry, false, false, nil, false, f.MaxFilenameLength)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -303,7 +303,7 @@ func (fsw *FilerStoreWrapper) prefixFilterEntries(ctx context.Context, dirPath u
|
||||
}
|
||||
}
|
||||
}
|
||||
if count < limit && lastFileName <= prefix && len(notPrefixed) == int(limit) {
|
||||
if count < limit && lastFileName <= prefix {
|
||||
notPrefixed = notPrefixed[:0]
|
||||
lastFileName, err = actualStore.ListDirectoryEntries(ctx, dirPath, lastFileName, false, limit, func(entry *Entry) bool {
|
||||
notPrefixed = append(notPrefixed, entry)
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
)
|
||||
|
||||
func TestCreateAndFind(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDBStore{}
|
||||
store.initialize(dir)
|
||||
@@ -32,7 +32,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
|
||||
t.Errorf("create entry %v: %v", entry1.FullPath, err)
|
||||
return
|
||||
}
|
||||
@@ -66,7 +66,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDBStore{}
|
||||
store.initialize(dir)
|
||||
@@ -88,7 +88,7 @@ func TestEmptyRoot(t *testing.T) {
|
||||
}
|
||||
|
||||
func BenchmarkInsertEntry(b *testing.B) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := b.TempDir()
|
||||
store := &LevelDBStore{}
|
||||
store.initialize(dir)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func TestCreateAndFind(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDB2Store{}
|
||||
store.initialize(dir, 2)
|
||||
@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
|
||||
t.Errorf("create entry %v: %v", entry1.FullPath, err)
|
||||
return
|
||||
}
|
||||
@@ -63,7 +63,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDB2Store{}
|
||||
store.initialize(dir, 2)
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
)
|
||||
|
||||
func TestCreateAndFind(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDB3Store{}
|
||||
store.initialize(dir)
|
||||
@@ -29,7 +29,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
|
||||
t.Errorf("create entry %v: %v", entry1.FullPath, err)
|
||||
return
|
||||
}
|
||||
@@ -63,7 +63,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestEmptyRoot(t *testing.T) {
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", nil)
|
||||
testFiler := filer.NewFiler(pb.ServerDiscovery{}, nil, "", "", "", "", "", 255, nil)
|
||||
dir := t.TempDir()
|
||||
store := &LevelDB3Store{}
|
||||
store.initialize(dir)
|
||||
|
||||
@@ -164,8 +164,8 @@ func (store *UniversalRedisStore) ListDirectoryEntries(ctx context.Context, dirP
|
||||
}
|
||||
|
||||
// sort
|
||||
slices.SortFunc(members, func(a, b string) bool {
|
||||
return strings.Compare(a, b) < 0
|
||||
slices.SortFunc(members, func(a, b string) int {
|
||||
return strings.Compare(a, b)
|
||||
})
|
||||
|
||||
// limit
|
||||
|
||||
@@ -34,7 +34,7 @@ func TestCreateAndFind(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false); err != nil {
|
||||
if err := testFiler.CreateEntry(ctx, entry1, false, false, nil, false, testFiler.MaxFilenameLength); err != nil {
|
||||
t.Errorf("create entry %v: %v", entry1.FullPath, err)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -45,11 +45,11 @@ func isSameChunks(a, b []*filer_pb.FileChunk) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) bool {
|
||||
return strings.Compare(i.ETag, j.ETag) < 0
|
||||
slices.SortFunc(a, func(i, j *filer_pb.FileChunk) int {
|
||||
return strings.Compare(i.ETag, j.ETag)
|
||||
})
|
||||
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) bool {
|
||||
return strings.Compare(i.ETag, j.ETag) < 0
|
||||
slices.SortFunc(b, func(i, j *filer_pb.FileChunk) int {
|
||||
return strings.Compare(i.ETag, j.ETag)
|
||||
})
|
||||
for i := 0; i < len(a); i++ {
|
||||
if a[i].ETag != b[i].ETag {
|
||||
|
||||
@@ -228,65 +228,32 @@ func (store *TikvStore) ListDirectoryPrefixedEntries(ctx context.Context, dirPat
|
||||
return err
|
||||
}
|
||||
defer iter.Close()
|
||||
i := int64(0)
|
||||
first := true
|
||||
for iter.Valid() {
|
||||
if first {
|
||||
first = false
|
||||
if !includeStartFile {
|
||||
if iter.Valid() {
|
||||
// Check first item is lastFileStart
|
||||
if bytes.Equal(iter.Key(), lastFileStart) {
|
||||
// Is lastFileStart and not include start file, just
|
||||
// ignore it.
|
||||
err = iter.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check for limitation
|
||||
if limit > 0 {
|
||||
i++
|
||||
if i > limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Validate key prefix
|
||||
for i := int64(0); i < limit && iter.Valid(); i++ {
|
||||
key := iter.Key()
|
||||
if !bytes.HasPrefix(key, directoryPrefix) {
|
||||
break
|
||||
}
|
||||
value := iter.Value()
|
||||
|
||||
// Start process
|
||||
fileName := getNameFromKey(key)
|
||||
if fileName != "" {
|
||||
// Got file name, then generate the Entry
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.NewFullPath(string(dirPath), fileName),
|
||||
}
|
||||
// Update lastFileName
|
||||
lastFileName = fileName
|
||||
// Check for decode value.
|
||||
if decodeErr := entry.DecodeAttributesAndChunks(value); decodeErr != nil {
|
||||
// Got error just return the error
|
||||
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
|
||||
return err
|
||||
}
|
||||
// Run for each callback if return false just break the iteration
|
||||
if !eachEntryFunc(entry) {
|
||||
if fileName == "" || fileName == startFileName && !includeStartFile {
|
||||
if err := iter.Next(); err != nil {
|
||||
break
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
// End process
|
||||
lastFileName = fileName
|
||||
entry := &filer.Entry{
|
||||
FullPath: util.NewFullPath(string(dirPath), fileName),
|
||||
}
|
||||
|
||||
err = iter.Next()
|
||||
if err != nil {
|
||||
return err
|
||||
// println("list", entry.FullPath, "chunks", len(entry.GetChunks()))
|
||||
if decodeErr := entry.DecodeAttributesAndChunks(util.MaybeDecompressData(iter.Value())); decodeErr != nil {
|
||||
err = decodeErr
|
||||
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
|
||||
break
|
||||
}
|
||||
if err := iter.Next(); !eachEntryFunc(entry) || err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -150,7 +150,10 @@ func (i *InodeToPath) MarkChildrenCached(fullpath util.FullPath) {
|
||||
defer i.RUnlock()
|
||||
inode, found := i.path2inode[fullpath]
|
||||
if !found {
|
||||
glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
|
||||
// https://github.com/seaweedfs/seaweedfs/issues/4968
|
||||
// glog.Fatalf("MarkChildrenCached not found inode %v", fullpath)
|
||||
glog.Warningf("MarkChildrenCached not found inode %v", fullpath)
|
||||
return
|
||||
}
|
||||
path, found := i.inode2path[inode]
|
||||
path.isChildrenCached = true
|
||||
|
||||
@@ -57,11 +57,10 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
|
||||
}
|
||||
|
||||
var clientEpoch int32
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "mount",
|
||||
ClientId: selfSignature,
|
||||
ClientEpoch: clientEpoch,
|
||||
ClientEpoch: 1,
|
||||
SelfSignature: selfSignature,
|
||||
PathPrefix: dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
@@ -70,8 +69,8 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
util.RetryForever("followMetaUpdates", func() error {
|
||||
clientEpoch++
|
||||
util.RetryUntil("followMetaUpdates", func() error {
|
||||
metadataFollowOption.ClientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, processEventFn)
|
||||
}, func(err error) bool {
|
||||
glog.Errorf("follow metadata updates: %v", err)
|
||||
|
||||
@@ -43,7 +43,7 @@ func Test_allocateOneBroker(t *testing.T) {
|
||||
|
||||
type args struct {
|
||||
brokers cmap.ConcurrentMap[string, *BrokerStats]
|
||||
partitionCount int
|
||||
partitionCount int32
|
||||
}
|
||||
|
||||
func testThem(t *testing.T, tests []struct {
|
||||
|
||||
@@ -3,6 +3,7 @@ package balancer
|
||||
import (
|
||||
"fmt"
|
||||
cmap "github.com/orcaman/concurrent-map/v2"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
)
|
||||
|
||||
@@ -11,6 +12,23 @@ const (
|
||||
LockBrokerBalancer = "broker_balancer"
|
||||
)
|
||||
|
||||
// Balancer collects stats from all brokers.
|
||||
//
|
||||
// When publishers wants to create topics, it picks brokers to assign the topic partitions.
|
||||
// When consumers wants to subscribe topics, it tells which brokers are serving the topic partitions.
|
||||
//
|
||||
// When a partition needs to be split or merged, or a partition needs to be moved to another broker,
|
||||
// the balancer will let the broker tell the consumer instance to stop processing the partition.
|
||||
// The existing consumer instance will flush the internal state, and then stop processing.
|
||||
// Then the balancer will tell the brokers to start sending new messages in the new/moved partition to the consumer instances.
|
||||
//
|
||||
// Failover to standby consumer instances:
|
||||
//
|
||||
// A consumer group can have min and max number of consumer instances.
|
||||
// For consumer instances joined after the max number, they will be in standby mode.
|
||||
//
|
||||
// When a consumer instance is down, the broker will notice this and inform the balancer.
|
||||
// The balancer will then tell the broker to send the partition to another standby consumer instance.
|
||||
type Balancer struct {
|
||||
Brokers cmap.ConcurrentMap[string, *BrokerStats]
|
||||
}
|
||||
@@ -30,11 +48,9 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
|
||||
currentTopicPartitions := bs.Stats.Items()
|
||||
for _, topicPartitionStats := range stats.Stats {
|
||||
tps := &TopicPartitionStats{
|
||||
TopicPartition: TopicPartition{
|
||||
Namespace: topicPartitionStats.Topic.Namespace,
|
||||
Topic: topicPartitionStats.Topic.Name,
|
||||
RangeStart: topicPartitionStats.Partition.RangeStart,
|
||||
RangeStop: topicPartitionStats.Partition.RangeStop,
|
||||
TopicPartition: topic.TopicPartition{
|
||||
Topic: topic.Topic{Namespace: topicPartitionStats.Topic.Namespace, Name: topicPartitionStats.Topic.Name},
|
||||
Partition: topic.Partition{RangeStart: topicPartitionStats.Partition.RangeStart, RangeStop: topicPartitionStats.Partition.RangeStop, RingSize: topicPartitionStats.Partition.RingSize},
|
||||
},
|
||||
ConsumerCount: topicPartitionStats.ConsumerCount,
|
||||
IsLeader: topicPartitionStats.IsLeader,
|
||||
@@ -52,20 +68,26 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) {
|
||||
|
||||
}
|
||||
|
||||
func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Partition) {
|
||||
tps := &TopicPartitionStats{
|
||||
TopicPartition: topic.TopicPartition{
|
||||
Topic: topic.Topic{Namespace: t.Namespace, Name: t.Name},
|
||||
Partition: topic.Partition{RangeStart: partition.RangeStart, RangeStop: partition.RangeStop},
|
||||
},
|
||||
ConsumerCount: 0,
|
||||
IsLeader: true,
|
||||
}
|
||||
key := tps.TopicPartition.String()
|
||||
bs.Stats.Set(key, tps)
|
||||
}
|
||||
|
||||
func (bs *BrokerStats) String() string {
|
||||
return fmt.Sprintf("BrokerStats{TopicPartitionCount:%d, ConsumerCount:%d, CpuUsagePercent:%d, Stats:%+v}",
|
||||
bs.TopicPartitionCount, bs.ConsumerCount, bs.CpuUsagePercent, bs.Stats.Items())
|
||||
}
|
||||
|
||||
type TopicPartition struct {
|
||||
Namespace string
|
||||
Topic string
|
||||
RangeStart int32
|
||||
RangeStop int32
|
||||
}
|
||||
|
||||
type TopicPartitionStats struct {
|
||||
TopicPartition
|
||||
topic.TopicPartition
|
||||
ConsumerCount int32
|
||||
IsLeader bool
|
||||
}
|
||||
@@ -81,7 +103,3 @@ func NewBrokerStats() *BrokerStats {
|
||||
Stats: cmap.New[*TopicPartitionStats](),
|
||||
}
|
||||
}
|
||||
|
||||
func (tp *TopicPartition) String() string {
|
||||
return fmt.Sprintf("%v.%v-%04d-%04d", tp.Namespace, tp.Topic, tp.RangeStart, tp.RangeStop)
|
||||
}
|
||||
@@ -16,7 +16,7 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
|
||||
for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
|
||||
topicPartitionStat := topicPartitionStatsItem.Val
|
||||
if topicPartitionStat.TopicPartition.Namespace == topic.Namespace &&
|
||||
topicPartitionStat.TopicPartition.Topic == topic.Name {
|
||||
topicPartitionStat.TopicPartition.Name == topic.Name {
|
||||
assignment := &mq_pb.BrokerPartitionAssignment{
|
||||
Partition: &mq_pb.Partition{
|
||||
RingSize: MaxPartitionCount,
|
||||
@@ -24,11 +24,8 @@ func (b *Balancer) LookupOrAllocateTopicPartitions(topic *mq_pb.Topic, publish b
|
||||
RangeStop: topicPartitionStat.RangeStop,
|
||||
},
|
||||
}
|
||||
if topicPartitionStat.IsLeader {
|
||||
assignment.LeaderBroker = broker
|
||||
} else {
|
||||
assignment.FollowerBrokers = append(assignment.FollowerBrokers, broker)
|
||||
}
|
||||
// TODO fix follower setting
|
||||
assignment.LeaderBroker = broker
|
||||
assignments = append(assignments, assignment)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// BrokerConnectToBalancer receives connections from brokers and collects stats
|
||||
// ConnectToBalancer receives connections from brokers and collects stats
|
||||
func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessaging_ConnectToBalancerServer) error {
|
||||
if !broker.lockAsBalancer.IsLocked() {
|
||||
return status.Errorf(codes.Unavailable, "not current broker balancer")
|
||||
@@ -20,9 +20,16 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
|
||||
|
||||
// process init message
|
||||
initMessage := req.GetInit()
|
||||
brokerStats := balancer.NewBrokerStats()
|
||||
var brokerStats *balancer.BrokerStats
|
||||
if initMessage != nil {
|
||||
broker.Balancer.Brokers.Set(initMessage.Broker, brokerStats)
|
||||
var found bool
|
||||
brokerStats, found = broker.Balancer.Brokers.Get(initMessage.Broker)
|
||||
if !found {
|
||||
brokerStats = balancer.NewBrokerStats()
|
||||
if !broker.Balancer.Brokers.SetIfAbsent(initMessage.Broker, brokerStats) {
|
||||
brokerStats, _ = broker.Balancer.Brokers.Get(initMessage.Broker)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return status.Errorf(codes.InvalidArgument, "balancer init message is empty")
|
||||
}
|
||||
@@ -42,8 +49,8 @@ func (broker *MessageQueueBroker) ConnectToBalancer(stream mq_pb.SeaweedMessagin
|
||||
if receivedStats := req.GetStats(); receivedStats != nil {
|
||||
brokerStats.UpdateStats(receivedStats)
|
||||
|
||||
glog.V(3).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
|
||||
glog.V(3).Infof("received stats: %+v", receivedStats)
|
||||
glog.V(4).Infof("broker %s stats: %+v", initMessage.Broker, brokerStats)
|
||||
glog.V(4).Infof("received stats: %+v", receivedStats)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/balancer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
@@ -9,14 +11,14 @@ import (
|
||||
"google.golang.org/grpc/status"
|
||||
)
|
||||
|
||||
// CreateTopic Runs on any broker, but proxied to the balancer if not the balancer
|
||||
func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_pb.CreateTopicRequest) (resp *mq_pb.CreateTopicResponse, err error) {
|
||||
// ConfigureTopic Runs on any broker, but proxied to the balancer if not the balancer
|
||||
func (broker *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb.ConfigureTopicRequest) (resp *mq_pb.ConfigureTopicResponse, err error) {
|
||||
if broker.currentBalancer == "" {
|
||||
return nil, status.Errorf(codes.Unavailable, "no balancer")
|
||||
}
|
||||
if !broker.lockAsBalancer.IsLocked() {
|
||||
proxyErr := broker.withBrokerClient(false, broker.currentBalancer, func(client mq_pb.SeaweedMessagingClient) error {
|
||||
resp, err = client.CreateTopic(ctx, request)
|
||||
resp, err = client.ConfigureTopic(ctx, request)
|
||||
return nil
|
||||
})
|
||||
if proxyErr != nil {
|
||||
@@ -25,26 +27,40 @@ func (broker *MessageQueueBroker) CreateTopic(ctx context.Context, request *mq_p
|
||||
return resp, err
|
||||
}
|
||||
|
||||
ret := &mq_pb.CreateTopicResponse{}
|
||||
ret := &mq_pb.ConfigureTopicResponse{}
|
||||
ret.BrokerPartitionAssignments, err = broker.Balancer.LookupOrAllocateTopicPartitions(request.Topic, true, request.PartitionCount)
|
||||
|
||||
for _, bpa := range ret.BrokerPartitionAssignments {
|
||||
// fmt.Printf("create topic %s on %s\n", request.Topic, bpa.LeaderBroker)
|
||||
if doCreateErr := broker.withBrokerClient(false, pb.ServerAddress(bpa.LeaderBroker), func(client mq_pb.SeaweedMessagingClient) error {
|
||||
_, doCreateErr := client.DoCreateTopic(ctx, &mq_pb.DoCreateTopicRequest{
|
||||
_, doCreateErr := client.DoConfigureTopic(ctx, &mq_pb.DoConfigureTopicRequest{
|
||||
Topic: request.Topic,
|
||||
Partition: bpa.Partition,
|
||||
})
|
||||
return doCreateErr
|
||||
if doCreateErr != nil {
|
||||
return fmt.Errorf("do create topic %s on %s: %v", request.Topic, bpa.LeaderBroker, doCreateErr)
|
||||
}
|
||||
brokerStats, found := broker.Balancer.Brokers.Get(bpa.LeaderBroker)
|
||||
if !found {
|
||||
brokerStats = balancer.NewBrokerStats()
|
||||
if !broker.Balancer.Brokers.SetIfAbsent(bpa.LeaderBroker, brokerStats) {
|
||||
brokerStats, _ = broker.Balancer.Brokers.Get(bpa.LeaderBroker)
|
||||
}
|
||||
}
|
||||
brokerStats.RegisterAssignment(request.Topic, bpa.Partition)
|
||||
return nil
|
||||
}); doCreateErr != nil {
|
||||
return nil, doCreateErr
|
||||
}
|
||||
}
|
||||
|
||||
// TODO revert if some error happens in the middle of the assignments
|
||||
|
||||
return ret, err
|
||||
}
|
||||
|
||||
func (broker *MessageQueueBroker) DoCreateTopic(ctx context.Context, req *mq_pb.DoCreateTopicRequest) (resp *mq_pb.DoCreateTopicResponse, err error) {
|
||||
ret := &mq_pb.DoCreateTopicResponse{}
|
||||
func (broker *MessageQueueBroker) DoConfigureTopic(ctx context.Context, req *mq_pb.DoConfigureTopicRequest) (resp *mq_pb.DoConfigureTopicResponse, err error) {
|
||||
ret := &mq_pb.DoConfigureTopicResponse{}
|
||||
t, p := topic.FromPbTopic(req.Topic), topic.FromPbPartition(req.Partition)
|
||||
localTopicPartition := broker.localTopicManager.GetTopicPartition(t, p)
|
||||
if localTopicPartition == nil {
|
||||
|
||||
@@ -2,6 +2,7 @@ package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
@@ -62,18 +63,20 @@ func (broker *MessageQueueBroker) ListTopics(ctx context.Context, request *mq_pb
|
||||
}
|
||||
|
||||
ret := &mq_pb.ListTopicsResponse{}
|
||||
knownTopics := make(map[*mq_pb.Topic]struct{})
|
||||
knownTopics := make(map[string]struct{})
|
||||
for brokerStatsItem := range broker.Balancer.Brokers.IterBuffered() {
|
||||
_, brokerStats := brokerStatsItem.Key, brokerStatsItem.Val
|
||||
for topicPartitionStatsItem := range brokerStats.Stats.IterBuffered() {
|
||||
topicPartitionStat := topicPartitionStatsItem.Val
|
||||
topic := &mq_pb.Topic{
|
||||
Namespace: topicPartitionStat.TopicPartition.Namespace,
|
||||
Name: topicPartitionStat.TopicPartition.Topic,
|
||||
Name: topicPartitionStat.TopicPartition.Name,
|
||||
}
|
||||
if _, found := knownTopics[topic]; found {
|
||||
topicKey := fmt.Sprintf("%s/%s", topic.Namespace, topic.Name)
|
||||
if _, found := knownTopics[topicKey]; found {
|
||||
continue
|
||||
}
|
||||
knownTopics[topicKey] = struct{}{}
|
||||
ret.Topics = append(ret.Topics, topic)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,8 +11,8 @@ import (
|
||||
|
||||
func (broker *MessageQueueBroker) Subscribe(req *mq_pb.SubscribeRequest, stream mq_pb.SeaweedMessaging_SubscribeServer) error {
|
||||
|
||||
localTopicPartition := broker.localTopicManager.GetTopicPartition(topic.FromPbTopic(req.Cursor.Topic),
|
||||
topic.FromPbPartition(req.Cursor.Partition))
|
||||
localTopicPartition := broker.localTopicManager.GetTopicPartition(topic.FromPbTopic(req.GetInit().Topic),
|
||||
topic.FromPbPartition(req.GetInit().Partition))
|
||||
if localTopicPartition == nil {
|
||||
stream.Send(&mq_pb.SubscribeResponse{
|
||||
Message: &mq_pb.SubscribeResponse_Ctrl{
|
||||
@@ -24,7 +24,7 @@ func (broker *MessageQueueBroker) Subscribe(req *mq_pb.SubscribeRequest, stream
|
||||
return nil
|
||||
}
|
||||
|
||||
clientName := fmt.Sprintf("%s/%s-%s", req.Consumer.ConsumerGroup, req.Consumer.ConsumerId, req.Consumer.ClientId)
|
||||
clientName := fmt.Sprintf("%s/%s-%s", req.GetInit().ConsumerGroup, req.GetInit().ConsumerId, req.GetInit().ClientId)
|
||||
|
||||
localTopicPartition.Subscribe(clientName, time.Now(), func(logEntry *filer_pb.LogEntry) error {
|
||||
value := logEntry.GetData()
|
||||
|
||||
@@ -61,6 +61,7 @@ func (broker *MessageQueueBroker) BrokerConnectToBalancer(self string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("send stats message: %v", err)
|
||||
}
|
||||
glog.V(3).Infof("sent stats: %+v", stats)
|
||||
|
||||
time.Sleep(time.Millisecond*5000 + time.Duration(rand.Intn(1000))*time.Millisecond)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package broker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/test/bufconn"
|
||||
)
|
||||
|
||||
var lis *bufconn.Listener
|
||||
|
||||
func init() {
|
||||
lis = bufconn.Listen(1024 * 1024)
|
||||
server := grpc.NewServer()
|
||||
mq_pb.RegisterSeaweedMessagingServer(server, &MessageQueueBroker{})
|
||||
go func() {
|
||||
if err := server.Serve(lis); err != nil {
|
||||
fmt.Printf("Server exited with error: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func bufDialer(string, time.Duration) (net.Conn, error) {
|
||||
return lis.Dial()
|
||||
}
|
||||
|
||||
func TestMessageQueueBroker_ListTopics(t *testing.T) {
|
||||
conn, err := grpc.DialContext(context.Background(), "bufnet", grpc.WithDialer(bufDialer), grpc.WithInsecure())
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to dial bufnet: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
client := mq_pb.NewSeaweedMessagingClient(conn)
|
||||
request := &mq_pb.ListTopicsRequest{}
|
||||
|
||||
_, err = client.ListTopics(context.Background(), request)
|
||||
if err == nil {
|
||||
t.Fatalf("Add failed: %v", err)
|
||||
}
|
||||
|
||||
}
|
||||
@@ -22,11 +22,7 @@ func main() {
|
||||
Filter: "",
|
||||
}
|
||||
|
||||
subscriber := sub_client.NewTopicSubscriber(subscriberConfig, contentConfig)
|
||||
if err := subscriber.Connect("localhost:17777"); err != nil {
|
||||
fmt.Println(err)
|
||||
return
|
||||
}
|
||||
subscriber := sub_client.NewTopicSubscriber("localhost:17777", subscriberConfig, contentConfig)
|
||||
|
||||
subscriber.SetEachMessageFunc(func(key, value []byte) bool {
|
||||
println(string(key), "=>", string(value))
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package sub_client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func (sub *TopicSubscriber) doProcess() error {
|
||||
var wg sync.WaitGroup
|
||||
for _, brokerPartitionAssignment := range sub.brokerPartitionAssignments {
|
||||
brokerAddress := brokerPartitionAssignment.LeaderBroker
|
||||
grpcConnection, err := pb.GrpcDial(context.Background(), brokerAddress, true, sub.SubscriberConfig.GrpcDialOption)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial broker %s: %v", brokerAddress, err)
|
||||
}
|
||||
brokerClient := mq_pb.NewSeaweedMessagingClient(grpcConnection)
|
||||
subscribeClient, err := brokerClient.Subscribe(context.Background(), &mq_pb.SubscribeRequest{
|
||||
Message: &mq_pb.SubscribeRequest_Init{
|
||||
Init: &mq_pb.SubscribeRequest_InitMessage{
|
||||
ConsumerGroup: sub.SubscriberConfig.GroupId,
|
||||
ConsumerId: sub.SubscriberConfig.GroupInstanceId,
|
||||
Topic: &mq_pb.Topic{
|
||||
Namespace: sub.ContentConfig.Namespace,
|
||||
Name: sub.ContentConfig.Topic,
|
||||
},
|
||||
Partition: &mq_pb.Partition{
|
||||
RingSize: brokerPartitionAssignment.Partition.RingSize,
|
||||
RangeStart: brokerPartitionAssignment.Partition.RangeStart,
|
||||
RangeStop: brokerPartitionAssignment.Partition.RangeStop,
|
||||
},
|
||||
Filter: sub.ContentConfig.Filter,
|
||||
},
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create subscribe client: %v", err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if sub.OnCompletionFunc != nil {
|
||||
defer sub.OnCompletionFunc()
|
||||
}
|
||||
defer func() {
|
||||
subscribeClient.SendMsg(&mq_pb.SubscribeRequest{
|
||||
Message: &mq_pb.SubscribeRequest_Ack{
|
||||
Ack: &mq_pb.SubscribeRequest_AckMessage{
|
||||
Sequence: 0,
|
||||
},
|
||||
},
|
||||
})
|
||||
subscribeClient.CloseSend()
|
||||
}()
|
||||
for {
|
||||
resp, err := subscribeClient.Recv()
|
||||
if err != nil {
|
||||
fmt.Printf("subscribe error: %v\n", err)
|
||||
return
|
||||
}
|
||||
if resp.Message == nil {
|
||||
continue
|
||||
}
|
||||
switch m := resp.Message.(type) {
|
||||
case *mq_pb.SubscribeResponse_Data:
|
||||
if !sub.OnEachMessageFunc(m.Data.Key, m.Data.Value) {
|
||||
return
|
||||
}
|
||||
case *mq_pb.SubscribeResponse_Ctrl:
|
||||
if m.Ctrl.IsEndOfStream || m.Ctrl.IsEndOfTopic {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
@@ -1,72 +1,33 @@
|
||||
package sub_client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
|
||||
"sync"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"io"
|
||||
)
|
||||
|
||||
// Subscribe subscribes to a topic's specified partitions.
|
||||
// If a partition is moved to another broker, the subscriber will automatically reconnect to the new broker.
|
||||
|
||||
func (sub *TopicSubscriber) Subscribe() error {
|
||||
var wg sync.WaitGroup
|
||||
for _, brokerPartitionAssignment := range sub.brokerPartitionAssignments {
|
||||
brokerAddress := brokerPartitionAssignment.LeaderBroker
|
||||
grpcConnection, err := pb.GrpcDial(context.Background(), brokerAddress, true, sub.SubscriberConfig.GrpcDialOption)
|
||||
if err != nil {
|
||||
return fmt.Errorf("dial broker %s: %v", brokerAddress, err)
|
||||
util.RetryUntil("subscribe", func() error {
|
||||
// ask balancer for brokers of the topic
|
||||
if err := sub.doLookup(sub.bootstrapBroker); err != nil {
|
||||
return fmt.Errorf("lookup topic %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
|
||||
}
|
||||
brokerClient := mq_pb.NewSeaweedMessagingClient(grpcConnection)
|
||||
subscribeClient, err := brokerClient.Subscribe(context.Background(), &mq_pb.SubscribeRequest{
|
||||
Consumer: &mq_pb.SubscribeRequest_Consumer{
|
||||
ConsumerGroup: sub.SubscriberConfig.GroupId,
|
||||
ConsumerId: sub.SubscriberConfig.GroupInstanceId,
|
||||
},
|
||||
Cursor: &mq_pb.SubscribeRequest_Cursor{
|
||||
Topic: &mq_pb.Topic{
|
||||
Namespace: sub.ContentConfig.Namespace,
|
||||
Name: sub.ContentConfig.Topic,
|
||||
},
|
||||
Partition: &mq_pb.Partition{
|
||||
RingSize: brokerPartitionAssignment.Partition.RingSize,
|
||||
RangeStart: brokerPartitionAssignment.Partition.RangeStart,
|
||||
RangeStop: brokerPartitionAssignment.Partition.RangeStop,
|
||||
},
|
||||
Filter: sub.ContentConfig.Filter,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("create subscribe client: %v", err)
|
||||
// treat the first broker as the topic leader
|
||||
// connect to the leader broker
|
||||
|
||||
// subscribe to the topic
|
||||
if err := sub.doProcess(); err != nil {
|
||||
return fmt.Errorf("subscribe topic %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if sub.OnCompletionFunc != nil {
|
||||
defer sub.OnCompletionFunc()
|
||||
}
|
||||
for {
|
||||
resp, err := subscribeClient.Recv()
|
||||
if err != nil {
|
||||
fmt.Printf("subscribe error: %v\n", err)
|
||||
return
|
||||
}
|
||||
if resp.Message == nil {
|
||||
continue
|
||||
}
|
||||
switch m := resp.Message.(type) {
|
||||
case *mq_pb.SubscribeResponse_Data:
|
||||
if !sub.OnEachMessageFunc(m.Data.Key, m.Data.Value) {
|
||||
return
|
||||
}
|
||||
case *mq_pb.SubscribeResponse_Ctrl:
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}, func(err error) bool {
|
||||
if err == io.EOF {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -6,11 +6,13 @@ import (
|
||||
)
|
||||
|
||||
type SubscriberConfiguration struct {
|
||||
ClientId string
|
||||
GroupId string
|
||||
GroupInstanceId string
|
||||
BootstrapServers []string
|
||||
GrpcDialOption grpc.DialOption
|
||||
ClientId string
|
||||
GroupId string
|
||||
GroupInstanceId string
|
||||
GroupMinimumPeers int32
|
||||
GroupMaximumPeers int32
|
||||
BootstrapServers []string
|
||||
GrpcDialOption grpc.DialOption
|
||||
}
|
||||
|
||||
type ContentConfiguration struct {
|
||||
@@ -28,22 +30,17 @@ type TopicSubscriber struct {
|
||||
brokerPartitionAssignments []*mq_pb.BrokerPartitionAssignment
|
||||
OnEachMessageFunc OnEachMessageFunc
|
||||
OnCompletionFunc OnCompletionFunc
|
||||
bootstrapBroker string
|
||||
}
|
||||
|
||||
func NewTopicSubscriber(subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
|
||||
func NewTopicSubscriber(bootstrapBroker string, subscriber *SubscriberConfiguration, content *ContentConfiguration) *TopicSubscriber {
|
||||
return &TopicSubscriber{
|
||||
SubscriberConfig: subscriber,
|
||||
ContentConfig: content,
|
||||
bootstrapBroker: bootstrapBroker,
|
||||
}
|
||||
}
|
||||
|
||||
func (sub *TopicSubscriber) Connect(bootstrapBroker string) error {
|
||||
if err := sub.doLookup(bootstrapBroker); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sub *TopicSubscriber) SetEachMessageFunc(onEachMessageFn OnEachMessageFunc) {
|
||||
sub.OnEachMessageFunc = onEachMessageFn
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package coordinator
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
"sync"
|
||||
)
|
||||
|
||||
func (cg *ConsumerGroup) SetMinMaxActiveInstances(min, max int32) {
|
||||
cg.MinimumActiveInstances = min
|
||||
cg.MaximumActiveInstances = max
|
||||
}
|
||||
|
||||
func (cg *ConsumerGroup) AddConsumerGroupInstance(clientId string) *ConsumerGroupInstance {
|
||||
cgi := &ConsumerGroupInstance{
|
||||
ClientId: clientId,
|
||||
}
|
||||
cg.ConsumerGroupInstances.Set(clientId, cgi)
|
||||
return cgi
|
||||
}
|
||||
|
||||
func (cg *ConsumerGroup) RemoveConsumerGroupInstance(clientId string) {
|
||||
cg.ConsumerGroupInstances.Remove(clientId)
|
||||
}
|
||||
|
||||
func (cg *ConsumerGroup) CoordinateIfNeeded() {
|
||||
emptyInstanceCount, activeInstanceCount := int32(0), int32(0)
|
||||
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
|
||||
if cgi.Val.Partition == nil {
|
||||
// this consumer group instance is not assigned a partition
|
||||
// need to assign one
|
||||
emptyInstanceCount++
|
||||
} else {
|
||||
activeInstanceCount++
|
||||
}
|
||||
}
|
||||
|
||||
var delta int32
|
||||
if emptyInstanceCount > 0 {
|
||||
if cg.MinimumActiveInstances <= 0 {
|
||||
// need to assign more partitions
|
||||
delta = emptyInstanceCount
|
||||
} else if activeInstanceCount < cg.MinimumActiveInstances && activeInstanceCount+emptyInstanceCount >= cg.MinimumActiveInstances {
|
||||
// need to assign more partitions
|
||||
delta = cg.MinimumActiveInstances - activeInstanceCount
|
||||
}
|
||||
}
|
||||
|
||||
if cg.MaximumActiveInstances > 0 {
|
||||
if activeInstanceCount > cg.MaximumActiveInstances {
|
||||
// need to remove some partitions
|
||||
delta = cg.MaximumActiveInstances - activeInstanceCount
|
||||
}
|
||||
}
|
||||
if delta == 0 {
|
||||
return
|
||||
}
|
||||
cg.doCoordinate(activeInstanceCount + delta)
|
||||
}
|
||||
|
||||
func (cg *ConsumerGroup) doCoordinate(target int32) {
|
||||
// stop existing instances from processing
|
||||
var wg sync.WaitGroup
|
||||
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
|
||||
if cgi.Val.Partition != nil {
|
||||
wg.Add(1)
|
||||
go func(cgi *ConsumerGroupInstance) {
|
||||
defer wg.Done()
|
||||
// stop processing
|
||||
// flush internal state
|
||||
// wait for all messages to be processed
|
||||
// close the connection
|
||||
}(cgi.Val)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
partitions := topic.SplitPartitions(target)
|
||||
|
||||
// assign partitions to new instances
|
||||
i := 0
|
||||
for cgi := range cg.ConsumerGroupInstances.IterBuffered() {
|
||||
cgi.Val.Partition = partitions[i]
|
||||
i++
|
||||
wg.Add(1)
|
||||
go func(cgi *ConsumerGroupInstance) {
|
||||
defer wg.Done()
|
||||
// start processing
|
||||
// start consuming from the last offset
|
||||
}(cgi.Val)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package coordinator
|
||||
|
||||
import (
|
||||
cmap "github.com/orcaman/concurrent-map/v2"
|
||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||
)
|
||||
|
||||
type ConsumerGroupInstance struct {
|
||||
ClientId string
|
||||
// the consumer group instance may not have an active partition
|
||||
Partition *topic.Partition
|
||||
// processed message count
|
||||
ProcessedMessageCount int64
|
||||
}
|
||||
type ConsumerGroup struct {
|
||||
// map a client id to a consumer group instance
|
||||
ConsumerGroupInstances cmap.ConcurrentMap[string, *ConsumerGroupInstance]
|
||||
MinimumActiveInstances int32
|
||||
MaximumActiveInstances int32
|
||||
}
|
||||
type TopicConsumerGroups struct {
|
||||
// map a consumer group name to a consumer group
|
||||
ConsumerGroups cmap.ConcurrentMap[string, *ConsumerGroup]
|
||||
}
|
||||
|
||||
// Coordinator coordinates the instances in the consumer group for one topic.
|
||||
// It is responsible for:
|
||||
// 1. Assigning partitions to consumer instances.
|
||||
// 2. Reassigning partitions when a consumer instance is down.
|
||||
// 3. Reassigning partitions when a consumer instance is up.
|
||||
type Coordinator struct {
|
||||
// map client id to subscriber
|
||||
Subscribers cmap.ConcurrentMap[string, *ConsumerGroupInstance]
|
||||
// map topic name to consumer groups
|
||||
TopicSubscribers cmap.ConcurrentMap[string, map[string]TopicConsumerGroups]
|
||||
}
|
||||
@@ -19,7 +19,7 @@ func NewLocalTopicManager() *LocalTopicManager {
|
||||
}
|
||||
}
|
||||
|
||||
// AddTopic adds a topic to the local topic manager
|
||||
// AddTopicPartition adds a topic to the local topic manager
|
||||
func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition *LocalPartition) {
|
||||
localTopic, ok := manager.topics.Get(topic.String())
|
||||
if !ok {
|
||||
@@ -28,7 +28,9 @@ func (manager *LocalTopicManager) AddTopicPartition(topic Topic, localPartition
|
||||
Partitions: make([]*LocalPartition, 0),
|
||||
}
|
||||
}
|
||||
manager.topics.SetIfAbsent(topic.String(), localTopic)
|
||||
if !manager.topics.SetIfAbsent(topic.String(), localTopic) {
|
||||
localTopic, _ = manager.topics.Get(topic.String())
|
||||
}
|
||||
if localTopic.findPartition(localPartition.Partition) != nil {
|
||||
return
|
||||
}
|
||||
@@ -61,9 +63,26 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
|
||||
stats := &mq_pb.BrokerStats{
|
||||
Stats: make(map[string]*mq_pb.TopicPartitionStats),
|
||||
}
|
||||
|
||||
// collect current broker's cpu usage
|
||||
// this needs to be in front, so the following stats can be more accurate
|
||||
usages, err := cpu.Percent(duration, false)
|
||||
if err == nil && len(usages) > 0 {
|
||||
stats.CpuUsagePercent = int32(usages[0])
|
||||
}
|
||||
|
||||
// collect current broker's topics and partitions
|
||||
manager.topics.IterCb(func(topic string, localTopic *LocalTopic) {
|
||||
for _, localPartition := range localTopic.Partitions {
|
||||
stats.Stats[topic] = &mq_pb.TopicPartitionStats{
|
||||
topicPartition := &TopicPartition{
|
||||
Topic: Topic{Namespace: localTopic.Namespace, Name: localTopic.Name},
|
||||
Partition: Partition{
|
||||
RingSize: localPartition.RingSize,
|
||||
RangeStart: localPartition.RangeStart,
|
||||
RangeStop: localPartition.RangeStop,
|
||||
},
|
||||
}
|
||||
stats.Stats[topicPartition.String()] = &mq_pb.TopicPartitionStats{
|
||||
Topic: &mq_pb.Topic{
|
||||
Namespace: string(localTopic.Namespace),
|
||||
Name: localTopic.Name,
|
||||
@@ -75,15 +94,10 @@ func (manager *LocalTopicManager) CollectStats(duration time.Duration) *mq_pb.Br
|
||||
},
|
||||
ConsumerCount: localPartition.ConsumerCount,
|
||||
}
|
||||
// fmt.Printf("collect topic %+v partition %+v\n", topicPartition, localPartition.Partition)
|
||||
}
|
||||
})
|
||||
|
||||
// collect current broker's cpu usage
|
||||
usages, err := cpu.Percent(duration, false)
|
||||
if err == nil && len(usages) > 0 {
|
||||
stats.CpuUsagePercent = int32(usages[0])
|
||||
}
|
||||
|
||||
return stats
|
||||
|
||||
}
|
||||
@@ -30,3 +30,20 @@ func FromPbPartition(partition *mq_pb.Partition) Partition {
|
||||
RingSize: partition.RingSize,
|
||||
}
|
||||
}
|
||||
|
||||
func SplitPartitions(targetCount int32) []*Partition {
|
||||
partitions := make([]*Partition, 0, targetCount)
|
||||
partitionSize := PartitionCount / targetCount
|
||||
for i := int32(0); i < targetCount; i++ {
|
||||
partitionStop := (i + 1) * partitionSize
|
||||
if i == targetCount-1 {
|
||||
partitionStop = PartitionCount
|
||||
}
|
||||
partitions = append(partitions, &Partition{
|
||||
RangeStart: i * partitionSize,
|
||||
RangeStop: partitionStop,
|
||||
RingSize: PartitionCount,
|
||||
})
|
||||
}
|
||||
return partitions
|
||||
}
|
||||
@@ -7,14 +7,12 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
type Namespace string
|
||||
|
||||
type Topic struct {
|
||||
Namespace Namespace
|
||||
Namespace string
|
||||
Name string
|
||||
}
|
||||
|
||||
func NewTopic(namespace Namespace, name string) Topic {
|
||||
func NewTopic(namespace string, name string) Topic {
|
||||
return Topic{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
@@ -22,7 +20,7 @@ func NewTopic(namespace Namespace, name string) Topic {
|
||||
}
|
||||
func FromPbTopic(topic *mq_pb.Topic) Topic {
|
||||
return Topic{
|
||||
Namespace: Namespace(topic.Namespace),
|
||||
Namespace: topic.Namespace,
|
||||
Name: topic.Name,
|
||||
}
|
||||
}
|
||||
@@ -41,7 +39,7 @@ type Segment struct {
|
||||
func FromPbSegment(segment *mq_pb.Segment) *Segment {
|
||||
return &Segment{
|
||||
Topic: Topic{
|
||||
Namespace: Namespace(segment.Namespace),
|
||||
Namespace: segment.Namespace,
|
||||
Name: segment.Topic,
|
||||
},
|
||||
Id: segment.Id,
|
||||
|
||||
Loaded 100 of 191 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user