mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
286
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2c4c2f0994 | ||
|
|
f193942adc | ||
|
|
61c42f9991 | ||
|
|
aafb874ad2 | ||
|
|
c4f44d56f7 | ||
|
|
f8aa5ea844 | ||
|
|
13a8593ab9 | ||
|
|
06471dac9d | ||
|
|
5149b3d07b | ||
|
|
868f7875d7 | ||
|
|
26eff062c8 | ||
|
|
464a71a373 | ||
|
|
ee4f7cd636 | ||
|
|
3fd659df2a | ||
|
|
0d9ba8c612 | ||
|
|
0515014149 | ||
|
|
a82c44972b | ||
|
|
14a19170e2 | ||
|
|
84ced2abf5 | ||
|
|
fe1716ccdf | ||
|
|
1ce725728d | ||
|
|
11549706ef | ||
|
|
8ec1bc2c99 | ||
|
|
f97e663f0d | ||
|
|
656283519b | ||
|
|
88b292fb3f | ||
|
|
ccec3acd5b | ||
|
|
149082f8ac | ||
|
|
f1883a9975 | ||
|
|
63b415bdf4 | ||
|
|
152fc4890a | ||
|
|
3b88ab42aa | ||
|
|
44b9d72ef0 | ||
|
|
572cc440ce | ||
|
|
94fbf02ea2 | ||
|
|
75f7893c11 | ||
|
|
8f99e1defe | ||
|
|
b6a516fed6 | ||
|
|
0b0fb9b9e4 | ||
|
|
1e22d5caf2 | ||
|
|
4dd890d4a2 | ||
|
|
3fbf4f6189 | ||
|
|
4527ead295 | ||
|
|
a25bca0692 | ||
|
|
25535e9c36 | ||
|
|
2e42cdeb22 | ||
|
|
5abb78bbd4 | ||
|
|
c0a0b84c66 | ||
|
|
a5740619e1 | ||
|
|
273d677ebd | ||
|
|
afbef0044a | ||
|
|
609902f0cf | ||
|
|
cd3bc35b8d | ||
|
|
2c14e9ac9f | ||
|
|
476022c2dc | ||
|
|
68feffa4e6 | ||
|
|
5ee04d20fa | ||
|
|
670e2185e4 | ||
|
|
d6af3a8eb6 | ||
|
|
5caff2c0f0 | ||
|
|
9bdbf9c880 | ||
|
|
0bffb3770f | ||
|
|
abef448c51 | ||
|
|
65b4dc6448 | ||
|
|
19fd0f7d5b | ||
|
|
0bd4f5ea61 | ||
|
|
17b0a94419 | ||
|
|
5230bb3656 | ||
|
|
a6b360b9f1 | ||
|
|
6812a5933c | ||
|
|
26f15d0079 | ||
|
|
fb4b61036c | ||
|
|
a0931be0c0 | ||
|
|
5aec6da8a3 | ||
|
|
e23f3d6eca | ||
|
|
70fc8d06f3 | ||
|
|
ca7cc61319 | ||
|
|
b0da8788a1 | ||
|
|
68c3cf767c | ||
|
|
d1d1757520 | ||
|
|
af0727ae4b | ||
|
|
a09a7c53e7 | ||
|
|
38dc1259f9 | ||
|
|
0271eb8732 | ||
|
|
684ef483f1 | ||
|
|
e7f4418855 | ||
|
|
ddd1de1beb | ||
|
|
e32422b971 | ||
|
|
26fb6deb4e | ||
|
|
eacf808877 | ||
|
|
5669d29d04 | ||
|
|
4310e1fac4 | ||
|
|
8fffe3e822 | ||
|
|
8f3bd202e5 | ||
|
|
9187a64975 | ||
|
|
1ebb549f77 | ||
|
|
a90b777ff4 | ||
|
|
17e91d2917 | ||
|
|
b7f011f777 | ||
|
|
65484e80ef | ||
|
|
64b2fbe0f1 | ||
|
|
2c347bec0c | ||
|
|
d77c787ec6 | ||
|
|
7034fd155f | ||
|
|
7d9f9ecda5 | ||
|
|
79d42eede4 | ||
|
|
9204ee2d2d | ||
|
|
2e351aa967 | ||
|
|
b0062c23cb | ||
|
|
51d9066e90 | ||
|
|
d999f1f0e2 | ||
|
|
e624a1c234 | ||
|
|
d232590299 | ||
|
|
953c7d648f | ||
|
|
3297d59360 | ||
|
|
1a97e1ddd8 | ||
|
|
25b237b83c | ||
|
|
59f55c1a66 | ||
|
|
b2a3860290 | ||
|
|
4511edc871 | ||
|
|
7592d013fe | ||
|
|
6fdff0bb18 | ||
|
|
c2129d5555 | ||
|
|
fdf1064a92 | ||
|
|
dc283e3bd9 | ||
|
|
00e243d98e | ||
|
|
b6dd4b8477 | ||
|
|
42766bccc8 | ||
|
|
e9fda774f4 | ||
|
|
4867aa03ab | ||
|
|
d75a7b7f62 | ||
|
|
fafff5896e | ||
|
|
a36967982d | ||
|
|
3e714a53cf | ||
|
|
5a46005ddf | ||
|
|
54440f0d63 | ||
|
|
fab0ab5440 | ||
|
|
f352616b7d | ||
|
|
18686b7375 | ||
|
|
68f1cc34a5 | ||
|
|
e1b92bc701 | ||
|
|
ff7b6d779e | ||
|
|
2e240704ab | ||
|
|
19245dde50 | ||
|
|
4131874fa8 | ||
|
|
9f55c7c90e | ||
|
|
a408b46d95 | ||
|
|
0a22eea55d | ||
|
|
d50d736e68 | ||
|
|
ae9388723f | ||
|
|
31db90942b | ||
|
|
839dd438bc | ||
|
|
bf3d0c4850 | ||
|
|
ea5bbf30bb | ||
|
|
a9b4be50af | ||
|
|
8ecdf958ab | ||
|
|
0f653d2153 | ||
|
|
0122e022ea | ||
|
|
5614ad0000 | ||
|
|
e60a7af644 | ||
|
|
88b2bbadd1 | ||
|
|
095cdb0c7f | ||
|
|
44ad07276c | ||
|
|
ccc030b245 | ||
|
|
130bc3e668 | ||
|
|
d1ce89950e | ||
|
|
da1d3b5a33 | ||
|
|
b5b5c67cf6 | ||
|
|
1d6a348d55 | ||
|
|
04a9dc898f | ||
|
|
adf80dc3f6 | ||
|
|
2d213d87e1 | ||
|
|
efef6e94bf | ||
|
|
d4dec6c730 | ||
|
|
080bfa7052 | ||
|
|
0f2a8b0af1 | ||
|
|
5aa89844b1 | ||
|
|
671ee6af6d | ||
|
|
422bfaed69 | ||
|
|
b9ab65dba5 | ||
|
|
2b82d0f871 | ||
|
|
8e80ce9476 | ||
|
|
ff649d615d | ||
|
|
3e69a3aac5 | ||
|
|
04190f9951 | ||
|
|
d6ecde8a96 | ||
|
|
ddc5234f85 | ||
|
|
47c4227318 | ||
|
|
3cc7b90414 | ||
|
|
f1f14e28bc | ||
|
|
d54798c3ae | ||
|
|
6bfb91b539 | ||
|
|
bbd461b2c6 | ||
|
|
5db9fcccd4 | ||
|
|
de4545c28b | ||
|
|
5c783c16a1 | ||
|
|
fba05959ef | ||
|
|
8d8347d2f7 | ||
|
|
60c20dce99 | ||
|
|
279c2627f9 | ||
|
|
f90d23916e | ||
|
|
b2134b78d4 | ||
|
|
fa00882e81 | ||
|
|
f0b2650adb | ||
|
|
adb90bd252 | ||
|
|
5b43c4bb98 | ||
|
|
50dc2fe96b | ||
|
|
29d1312625 | ||
|
|
784daea1fa | ||
|
|
264be0d2d4 | ||
|
|
d695119073 | ||
|
|
57ab1f8516 | ||
|
|
dd71f54c6b | ||
|
|
71b33faef0 | ||
|
|
f5854d13df | ||
|
|
0454bb2d88 | ||
|
|
2268d38263 | ||
|
|
b12d9b8598 | ||
|
|
99c05a9c29 | ||
|
|
4d3a485992 | ||
|
|
b1cdea3503 | ||
|
|
2b0cb47a43 | ||
|
|
98bc93cf31 | ||
|
|
66dd114933 | ||
|
|
e17429223e | ||
|
|
bd70683871 | ||
|
|
59706c89fb | ||
|
|
7111c08289 | ||
|
|
c589e9837c | ||
|
|
244385bf0d | ||
|
|
889ecf5c9d | ||
|
|
0cf4f2e891 | ||
|
|
113f3b76f4 | ||
|
|
9ed7922bed | ||
|
|
e52bb3112d | ||
|
|
281f5ef6a5 | ||
|
|
c5b8a0554a | ||
|
|
cef493dd0d | ||
|
|
bd6f2716d3 | ||
|
|
0bf56298d5 | ||
|
|
8241c9a829 | ||
|
|
1f7934a89c | ||
|
|
6cf43f9f58 | ||
|
|
8cf75a152b | ||
|
|
9db15c28ec | ||
|
|
83d15b50ce | ||
|
|
32862b815b | ||
|
|
c5bcfd8942 | ||
|
|
ffbc6615bd | ||
|
|
1976ca9160 | ||
|
|
214b7cd286 | ||
|
|
f457956948 | ||
|
|
4daa791b45 | ||
|
|
db5515eada | ||
|
|
ef2f741823 | ||
|
|
2c4dbaedf2 | ||
|
|
673214574a | ||
|
|
3227e4175e | ||
|
|
439b79016d | ||
|
|
508de8f492 | ||
|
|
f86c0d7751 | ||
|
|
d531fffc89 | ||
|
|
e70cf681fe | ||
|
|
6b7cb33f54 | ||
|
|
a554419e45 | ||
|
|
28c36913b4 | ||
|
|
f05261157f | ||
|
|
e3c48e987b | ||
|
|
472528f6f0 | ||
|
|
cbff1dbd0e | ||
|
|
00fda86880 | ||
|
|
3f22a9db18 | ||
|
|
ca102a0b0b | ||
|
|
9bb0a9e306 | ||
|
|
545d5d0cc3 | ||
|
|
a9aa2d581f | ||
|
|
79af6b41d4 | ||
|
|
896c6b8d1a | ||
|
|
3d7baa30cd | ||
|
|
1134de8cbe | ||
|
|
292e98cc22 | ||
|
|
5869945f16 | ||
|
|
e037c71ec3 | ||
|
|
67b8c2853a | ||
|
|
302155a6fa | ||
|
|
5520b81551 |
No files matched your search
@@ -44,7 +44,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@0aa3e4818bacdeab4f9a97f768d3a54734a5852e # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -33,30 +33,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,30 +34,30 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Login to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_TOKEN }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -35,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -35,20 +35,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -20,7 +20,7 @@ jobs:
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
uses: docker/metadata-action@507c2f2dc502c992ad446e3d7a5dfbe311567a96 # v3
|
||||
uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175 # v3
|
||||
with:
|
||||
images: |
|
||||
chrislusf/seaweedfs
|
||||
@@ -34,20 +34,20 @@ jobs:
|
||||
org.opencontainers.image.vendor=Chris Lu
|
||||
-
|
||||
name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@ecf95283f03858871ff00b787d79c419715afc34 # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a # v1
|
||||
uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v1
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
-
|
||||
name: Build
|
||||
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 # v2
|
||||
uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825 # v2
|
||||
with:
|
||||
context: ./docker
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
|
||||
uses: actions/dependency-review-action@1360a344ccb0ab6e9475edef90ad2f46bf8003b1
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
@@ -0,0 +1,51 @@
|
||||
name: "helm: lint and test charts"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v3
|
||||
with:
|
||||
version: v3.10.0
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.4.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
run: |
|
||||
changed=$(ct list-changed --chart-dirs k8s/charts)
|
||||
if [[ -n "$changed" ]]; then
|
||||
echo "::set-output name=changed::true"
|
||||
fi
|
||||
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.7.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --all --chart-dirs k8s/charts
|
||||
@@ -8,7 +8,7 @@
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
|
||||
[](https://artifacthub.io/packages/search?repo=seaweedfs)
|
||||
|
||||

|
||||
|
||||
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
-->
|
||||
|
||||
### Gold Sponsors
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
|
||||
---
|
||||
@@ -66,6 +66,7 @@ Table of Contents
|
||||
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
|
||||
* [Compared to GlusterFS](#compared-to-glusterfs)
|
||||
* [Compared to Ceph](#compared-to-ceph)
|
||||
* [Compared to Minio](#compared-to-minio)
|
||||
* [Dev Plan](#dev-plan)
|
||||
* [Installation Guide](#installation-guide)
|
||||
* [Disk Related Topics](#disk-related-topics)
|
||||
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.20-buster as builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v7.5.3
|
||||
ENV ROCKSDB_VERSION v8.1.1
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
|
||||
@@ -3,7 +3,7 @@ FROM golang:1.20-buster as builder
|
||||
RUN apt-get update
|
||||
RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
|
||||
|
||||
ENV ROCKSDB_VERSION v7.5.3
|
||||
ENV ROCKSDB_VERSION v8.1.1
|
||||
|
||||
# build RocksDB
|
||||
RUN cd /tmp && \
|
||||
|
||||
+3
-2
@@ -29,7 +29,7 @@ go_build_large_disk:
|
||||
build_rocksdb_dev_env:
|
||||
docker build --no-cache -t chrislusf/rocksdb_dev_env -f Dockerfile.rocksdb_dev_env .
|
||||
|
||||
build_rocksdb_local:
|
||||
build_rocksdb_local: build_rocksdb_dev_env
|
||||
cd .. ; docker build --no-cache -t chrislusf/seaweedfs:rocksdb_local -f docker/Dockerfile.rocksdb_large_local .
|
||||
|
||||
build_rocksdb:
|
||||
@@ -62,7 +62,8 @@ k8s: build
|
||||
dev_registry: build
|
||||
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
|
||||
dev_replicate: build
|
||||
dev_replicate:
|
||||
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
|
||||
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
|
||||
dev_auditlog: build
|
||||
|
||||
@@ -3,22 +3,23 @@ module github.com/seaweedfs/seaweedfs
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.107.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.28.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
cloud.google.com/go v0.110.2 // indirect
|
||||
cloud.google.com/go/pubsub v1.31.0
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.44.189
|
||||
github.com/aws/aws-sdk-go v1.44.280
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.1
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.1 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
@@ -27,23 +28,23 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fclairamb/ftpserverlib v0.21.0
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.7.1
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/go-redsync/redsync/v4 v4.8.1
|
||||
github.com/go-sql-driver/mysql v1.7.1
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/gocql/gocql v1.5.2
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/protobuf v1.5.3
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.10.0 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -51,22 +52,22 @@ require (
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
github.com/jcmturner/gofork v1.7.6 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.3 // indirect
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect
|
||||
github.com/jinzhu/copier v0.3.5
|
||||
github.com/jmespath/go-jmespath v0.4.0 // indirect
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.14 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.6
|
||||
github.com/klauspost/compress v1.16.0 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.7
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.14
|
||||
github.com/lib/pq v1.10.9
|
||||
github.com/linxGnu/grocksdb v1.8.0
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.11 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -75,21 +76,21 @@ require (
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/posener/complete v1.2.3
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/pquerna/cachecontrol v0.2.0
|
||||
github.com/prometheus/client_golang v1.16.0
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0
|
||||
github.com/prometheus/common v0.42.0 // indirect
|
||||
github.com/prometheus/procfs v0.11.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/seaweedfs/raft v1.1.3
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.3 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/afero v1.9.5 // indirect
|
||||
github.com/spf13/cast v1.5.1 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.15.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/spf13/viper v1.16.0
|
||||
github.com/stretchr/testify v1.8.4
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.14.4
|
||||
@@ -104,141 +105,210 @@ require (
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.7
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.9
|
||||
go.mongodb.org/mongo-driver v1.11.6
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.28.0
|
||||
gocloud.dev/pubsub/natspubsub v0.28.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.3.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.5.0
|
||||
golang.org/x/oauth2 v0.2.0 // indirect
|
||||
golang.org/x/sys v0.4.0
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
golang.org/x/tools v0.5.0
|
||||
gocloud.dev v0.29.0
|
||||
gocloud.dev/pubsub/natspubsub v0.29.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.29.0
|
||||
golang.org/x/crypto v0.10.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
|
||||
golang.org/x/image v0.7.0
|
||||
golang.org/x/net v0.11.0
|
||||
golang.org/x/oauth2 v0.8.0 // indirect
|
||||
golang.org/x/sys v0.9.0
|
||||
golang.org/x/text v0.10.0 // indirect
|
||||
golang.org/x/tools v0.10.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.108.0
|
||||
google.golang.org/api v0.126.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||
google.golang.org/grpc v1.52.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/grpc v1.55.0
|
||||
google.golang.org/protobuf v1.30.0
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/libc v1.22.5 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.20.1
|
||||
modernc.org/sqlite v1.23.1
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.4.0
|
||||
github.com/arangodb/go-driver v1.4.1
|
||||
github.com/arangodb/go-driver v1.6.0
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.4
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.12
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.12
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.2
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.11
|
||||
github.com/hanwen/go-fuse/v2 v2.3.0
|
||||
github.com/hashicorp/raft v1.5.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/schollz/progressbar/v3 v3.13.0
|
||||
github.com/tikv/client-go/v2 v2.0.5
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
|
||||
github.com/rabbitmq/amqp091-go v1.8.1
|
||||
github.com/rclone/rclone v1.62.2
|
||||
github.com/schollz/progressbar/v3 v3.13.1
|
||||
github.com/tikv/client-go/v2 v2.0.7
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.14.0 // indirect
|
||||
cloud.google.com/go/compute v1.19.3 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v0.8.0 // indirect
|
||||
cloud.google.com/go/iam v1.0.1 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.2.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect
|
||||
github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect
|
||||
github.com/Max-Sum/base32768 v0.0.0-20230304063302-18e6ce5945fd // indirect
|
||||
github.com/Microsoft/go-winio v0.5.2 // indirect
|
||||
github.com/Unknwon/goconfig v1.0.0 // indirect
|
||||
github.com/abbot/go-http-auth v0.4.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.23 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/buengese/sgzip v0.1.1 // indirect
|
||||
github.com/calebcase/tmpfile v1.0.3 // indirect
|
||||
github.com/colinmarc/hdfs/v2 v2.3.0 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 // indirect
|
||||
github.com/elastic/gosigar v0.14.2 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/geoffgarside/ber v1.1.0 // indirect
|
||||
github.com/go-chi/chi/v5 v5.0.8 // indirect
|
||||
github.com/go-ole/go-ole v1.2.6 // indirect
|
||||
github.com/gofrs/flock v0.8.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/s2a-go v0.1.4 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.5.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/hirochachacha/go-smb2 v1.1.0 // indirect
|
||||
github.com/iguanesolutions/go-systemd/v5 v5.1.1 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/jlaffaye/ftp v0.1.1-0.20230214004652-d84bf4be2b6e // indirect
|
||||
github.com/jonboulle/clockwork v0.3.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/jtolio/eventkit v0.0.0-20221004135224-074cf276595b // indirect
|
||||
github.com/jzelinskie/whirlpool v0.0.0-20201016144138-0675e54bb004 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/koofr/go-httpclient v0.0.0-20230225102643-5d51a2e9dea6 // indirect
|
||||
github.com/koofr/go-koofrclient v0.0.0-20221207135200-cbd7fc9ad6a6 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/kylelemons/godebug v1.1.0 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/nats-io/nats.go v1.20.0 // indirect
|
||||
github.com/mitchellh/go-homedir v1.1.0 // indirect
|
||||
github.com/montanaflynn/stats v0.7.0 // indirect
|
||||
github.com/nats-io/nats.go v1.23.0 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/ncw/go-acd v0.0.0-20201019170801-fe55f33415b1 // indirect
|
||||
github.com/ncw/swift/v2 v2.0.1 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.6 // indirect
|
||||
github.com/oracle/oci-go-sdk/v65 v65.32.0 // indirect
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
|
||||
github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/rivo/uniseg v0.4.3 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
|
||||
github.com/pkg/sftp v1.13.6-0.20230213180117-971c283182b6 // indirect
|
||||
github.com/pkg/xattr v0.4.9 // indirect
|
||||
github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
|
||||
github.com/putdotio/go-putio/putio v0.0.0-20200123120452-16d982cac2b8 // indirect
|
||||
github.com/puzpuzpuz/xsync/v2 v2.4.1 // indirect
|
||||
github.com/rfjakob/eme v1.1.2 // indirect
|
||||
github.com/rivo/uniseg v0.4.4 // indirect
|
||||
github.com/shirou/gopsutil/v3 v3.23.2 // indirect
|
||||
github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
|
||||
github.com/sony/gobreaker v0.5.0 // indirect
|
||||
github.com/spacemonkeygo/monkit/v3 v3.0.19 // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stathat/consistent v1.0.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/t3rm1n4l/go-mega v0.0.0-20230228171823-a01a2cda13ca // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230329114254-1948c247c2b1 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/tklauser/go-sysconf v0.3.11 // indirect
|
||||
github.com/tklauser/numcpus v0.6.0 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect
|
||||
github.com/xanzy/ssh-agent v0.3.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
|
||||
github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
|
||||
github.com/yusufpapurcu/wmi v1.2.2 // indirect
|
||||
github.com/zeebo/blake3 v0.2.3 // indirect
|
||||
github.com/zeebo/errs v1.3.0 // indirect
|
||||
go.etcd.io/bbolt v1.3.7 // indirect
|
||||
go.etcd.io/etcd/api/v3 v3.5.9 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/term v0.4.0 // indirect
|
||||
golang.org/x/mod v0.11.0 // indirect
|
||||
golang.org/x/sync v0.3.0 // indirect
|
||||
golang.org/x/term v0.9.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
storj.io/common v0.0.0-20221123115229-fed3e6651b63 // indirect
|
||||
storj.io/drpc v0.0.32 // indirect
|
||||
storj.io/uplink v1.10.0 // indirect
|
||||
)
|
||||
|
||||
// replace github.com/seaweedfs/raft => /Users/chrislu/go/src/github.com/seaweedfs/raft
|
||||
@@ -0,0 +1,16 @@
|
||||
# Artifact Hub repository metadata file
|
||||
#
|
||||
# Some settings like the verified publisher flag or the ignored packages won't
|
||||
# be applied until the next time the repository is processed. Please keep in
|
||||
# mind that the repository won't be processed if it has not changed since the
|
||||
# last time it was processed. Depending on the repository kind, this is checked
|
||||
# in a different way. For Helm http based repositories, we consider it has
|
||||
# changed if the `index.yaml` file changes. For git based repositories, it does
|
||||
# when the hash of the last commit in the branch you set up changes. This does
|
||||
# NOT apply to ownership claim operations, which are processed immediately.
|
||||
#
|
||||
|
||||
repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
|
||||
# owners: # (optional, used to claim repository ownership)
|
||||
# - name: username
|
||||
# email: email
|
||||
File renamed without changes.
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.42"
|
||||
version: "3.42"
|
||||
appVersion: "3.53"
|
||||
version: "3.53"
|
||||
@@ -0,0 +1,81 @@
|
||||
# SEAWEEDFS - helm chart (2.x+)
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Add the helm repo
|
||||
|
||||
`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
|
||||
|
||||
### Install the helm chart
|
||||
|
||||
`helm install seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
### (Recommended) Provide `values.yaml`
|
||||
|
||||
`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
|
||||
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
|
||||
leveldb is the default database this only supports one filer replica.
|
||||
|
||||
To have multiple filers a external datastore is recommened.
|
||||
|
||||
Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
Here is an example:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server and the masters, all pods (master/volume/filer)
|
||||
should have anti-affinity rules to disallow running multiple component pods on the same host.
|
||||
|
||||
If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
On the volume stateful set added support for k8s PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
You can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels if applicable.
|
||||
|
||||
Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
|
||||
File renamed without changes.
File renamed without changes.
+1
-1
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-cert
|
||||
@@ -0,0 +1,10 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Issuer
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
namespace: {{ .Release.Namespace }}
|
||||
spec:
|
||||
ca:
|
||||
secretName: {{ template "seaweedfs.name" . }}-ca-cert
|
||||
{{- end }}
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: ClusterIssuer
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-clusterissuer
|
||||
+8
-6
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-client-cert
|
||||
@@ -7,10 +7,11 @@ metadata:
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-client-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-clusterissuer
|
||||
kind: ClusterIssuer
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
organization:
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ .Release.Namespace }}'
|
||||
@@ -26,8 +27,9 @@ spec:
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
keySize: {{ .Values.certificates.keySize }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
+8
-6
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-filer-cert
|
||||
@@ -7,10 +7,11 @@ metadata:
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-filer-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-clusterissuer
|
||||
kind: ClusterIssuer
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
organization:
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ .Release.Namespace }}'
|
||||
@@ -26,8 +27,9 @@ spec:
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
keySize: {{ .Values.certificates.keySize }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
File renamed without changes.
+12
@@ -20,6 +20,18 @@ spec:
|
||||
port: {{ .Values.filer.grpcPort }}
|
||||
targetPort: {{ .Values.filer.grpcPort }}
|
||||
protocol: TCP
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
- name: "swfs-s3"
|
||||
port: {{ .Values.filer.s3.port }}
|
||||
targetPort: {{ .Values.filer.s3.port }}
|
||||
protocol: TCP
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
- name: "swfs-s3-tls"
|
||||
port: {{ .Values.filer.s3.httpsPort }}
|
||||
targetPort: {{ .Values.filer.s3.httpsPort }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.filer.metricsPort }}
|
||||
File renamed without changes.
+19
-2
@@ -88,7 +88,12 @@ spec:
|
||||
{{- if .Values.filer.extraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.extraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
{{- if kindIs "string" $value }}
|
||||
value: {{ $value | quote }}
|
||||
{{- else }}
|
||||
valueFrom:
|
||||
{{ toYaml $value | nindent 16 | trim }}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.global.extraEnvironmentVars }}
|
||||
@@ -97,6 +102,12 @@ spec:
|
||||
value: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
valueFrom: {{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -142,6 +153,9 @@ spec:
|
||||
-encryptVolumeData \
|
||||
{{- end }}
|
||||
-ip=${POD_IP} \
|
||||
{{- if .Values.filer.filerGroup}}
|
||||
-filerGroup={{ .Values.filer.filerGroup}} \
|
||||
{{- end }}
|
||||
{{- if .Values.filer.s3.enabled }}
|
||||
-s3 \
|
||||
-s3.port={{ .Values.filer.s3.port }} \
|
||||
@@ -149,6 +163,9 @@ spec:
|
||||
-s3.domainName={{ .Values.filer.s3.domainName }} \
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if .Values.filer.s3.httpsPort }}
|
||||
-s3.port.https={{ .Values.filer.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
-s3.cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-s3.key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
@@ -162,14 +179,14 @@ spec:
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
- mountPath: /etc/sw
|
||||
name: config-users
|
||||
readOnly: true
|
||||
{{- if .Values.filer.enablePVC }}
|
||||
{{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
|
||||
- name: data-filer
|
||||
mountPath: /data
|
||||
{{- end }}
|
||||
+4
-1
@@ -14,8 +14,11 @@ metadata:
|
||||
{{ omit .Values.filer.ingress.annotations "kubernetes.io/ingress.class" | toYaml | nindent 4 }}
|
||||
spec:
|
||||
ingressClassName: {{ .Values.filer.ingress.className | quote }}
|
||||
tls:
|
||||
{{ .Values.filer.ingress.tls | default list | toYaml | nindent 6}}
|
||||
rules:
|
||||
- http:
|
||||
- host: {{ .Values.filer.ingress.host }}
|
||||
http:
|
||||
paths:
|
||||
- path: /sw-filer/?(.*)
|
||||
pathType: ImplementationSpecific
|
||||
+8
-6
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-master-cert
|
||||
@@ -7,10 +7,11 @@ metadata:
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-master-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-clusterissuer
|
||||
kind: ClusterIssuer
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
organization:
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ .Release.Namespace }}'
|
||||
@@ -26,8 +27,9 @@ spec:
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
keySize: {{ .Values.certificates.keySize }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
File renamed without changes.
File renamed without changes.
+2
-2
@@ -133,8 +133,8 @@ spec:
|
||||
{{- if .Values.master.garbageThreshold }}
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name : data-{{ .Release.Namespace }}
|
||||
mountPath: /data
|
||||
+4
-1
@@ -90,6 +90,9 @@ spec:
|
||||
-metricsPort {{ .Values.s3.metricsPort }} \
|
||||
{{- end }}
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
{{- if .Values.s3.httpsPort }}
|
||||
-port.https={{ .Values.s3.httpsPort }} \
|
||||
{{- end }}
|
||||
-cert.file=/usr/local/share/ca-certificates/client/tls.crt \
|
||||
-key.file=/usr/local/share/ca-certificates/client/tls.key \
|
||||
{{- end }}
|
||||
@@ -105,7 +108,7 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: "/logs/"
|
||||
+6
@@ -12,6 +12,12 @@ spec:
|
||||
port: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
|
||||
targetPort: {{ if .Values.s3.enabled }}{{ .Values.s3.port }}{{ else }}{{ .Values.filer.s3.port }}{{ end }}
|
||||
protocol: TCP
|
||||
{{- if and .Values.s3.enabled .Values.s3.httpsPort }}
|
||||
- name: "swfs-s3-tls"
|
||||
port: {{ .Values.s3.httpsPort }}
|
||||
targetPort: {{ .Values.s3.httpsPort }}
|
||||
protocol: TCP
|
||||
{{- end }}
|
||||
{{- if and .Values.s3.enabled .Values.s3.metricsPort }}
|
||||
- name: "metrics"
|
||||
port: {{ .Values.s3.metricsPort }}
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+8
-6
@@ -1,5 +1,5 @@
|
||||
{{- if .Values.global.enableSecurity }}
|
||||
apiVersion: certmanager.k8s.io/v1alpha1
|
||||
apiVersion: cert-manager.io/v1{{ if .Values.global.certificates.alphacrds }}alpha1{{ end }}
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: {{ template "seaweedfs.name" . }}-volume-cert
|
||||
@@ -7,10 +7,11 @@ metadata:
|
||||
spec:
|
||||
secretName: {{ template "seaweedfs.name" . }}-volume-cert
|
||||
issuerRef:
|
||||
name: {{ template "seaweedfs.name" . }}-clusterissuer
|
||||
kind: ClusterIssuer
|
||||
name: {{ template "seaweedfs.name" . }}-ca-issuer
|
||||
kind: Issuer
|
||||
commonName: {{ .Values.certificates.commonName }}
|
||||
organization:
|
||||
subject:
|
||||
organizations:
|
||||
- "SeaweedFS CA"
|
||||
dnsNames:
|
||||
- '*.{{ .Release.Namespace }}'
|
||||
@@ -26,8 +27,9 @@ spec:
|
||||
- {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
keyAlgorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
keySize: {{ .Values.certificates.keySize }}
|
||||
privateKey:
|
||||
algorithm: {{ .Values.certificates.keyAlgorithm }}
|
||||
size: {{ .Values.certificates.keySize }}
|
||||
duration: {{ .Values.certificates.duration }}
|
||||
renewBefore: {{ .Values.certificates.renewBefore }}
|
||||
{{- end }}
|
||||
File renamed without changes.
File renamed without changes.
+3
-3
@@ -138,9 +138,9 @@ spec:
|
||||
-fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
|
||||
{{- end }}
|
||||
-minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
|
||||
-compactionMBps={{ .Values.volume.compactionMBps }} \
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: "{{ .Values.volume.dir }}/"
|
||||
@@ -187,7 +187,7 @@ spec:
|
||||
port: {{ .Values.volume.port }}
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 90
|
||||
periodSeconds: 15
|
||||
successThreshold: 1
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 30
|
||||
@@ -5,10 +5,12 @@ global:
|
||||
repository: ""
|
||||
imageName: chrislusf/seaweedfs
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: imagepullsecret
|
||||
imagePullSecrets: ""
|
||||
restartPolicy: Always
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
certificates:
|
||||
alphacrds: false
|
||||
monitoring:
|
||||
enabled: false
|
||||
gatewayHost: null
|
||||
@@ -22,8 +24,8 @@ global:
|
||||
replicationPlacment: "001"
|
||||
extraEnvironmentVars:
|
||||
WEED_CLUSTER_DEFAULT: "sw"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
|
||||
|
||||
image:
|
||||
registry: ""
|
||||
@@ -44,11 +46,11 @@ master:
|
||||
volumePreallocate: false
|
||||
volumeSizeLimitMB: 1000
|
||||
loggingOverrideLevel: null
|
||||
#number of seconds between heartbeats, default 5
|
||||
# number of seconds between heartbeats, default 5
|
||||
pulseSeconds: null
|
||||
#threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
# threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
garbageThreshold: null
|
||||
#Prometheus push interval in seconds, default 15
|
||||
# Prometheus push interval in seconds, default 15
|
||||
metricsIntervalSec: 15
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
@@ -112,10 +114,10 @@ master:
|
||||
# nodeSelector labels for master pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to master pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -124,6 +126,7 @@ master:
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
host: "master.seaweedfs.local"
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/auth-type: "basic"
|
||||
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
|
||||
@@ -241,7 +244,7 @@ volume:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: null
|
||||
resources: {}
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -251,10 +254,10 @@ volume:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-volume: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-volume: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -273,6 +276,7 @@ filer:
|
||||
grpcPort: 18888
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
filerGroup: ""
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
# Y number of replica in other racks in the same data center
|
||||
@@ -358,10 +362,10 @@ filer:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -370,7 +374,9 @@ filer:
|
||||
ingress:
|
||||
enabled: false
|
||||
className: "nginx"
|
||||
host: "seaweedfs.cluster.local"
|
||||
annotations:
|
||||
nginx.ingress.kubernetes.io/backend-protocol: GRPC
|
||||
nginx.ingress.kubernetes.io/auth-type: "basic"
|
||||
nginx.ingress.kubernetes.io/auth-secret: "default/ingress-basic-auth-secret"
|
||||
nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - SW-Filer'
|
||||
@@ -389,7 +395,7 @@ filer:
|
||||
|
||||
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
|
||||
extraEnvironmentVars:
|
||||
WEED_MYSQL_ENABLED: "true"
|
||||
WEED_MYSQL_ENABLED: "false"
|
||||
WEED_MYSQL_HOSTNAME: "mysql-db-host"
|
||||
WEED_MYSQL_PORT: "3306"
|
||||
WEED_MYSQL_DATABASE: "sw_database"
|
||||
@@ -400,17 +406,31 @@ filer:
|
||||
# enable usage of memsql as filer backend
|
||||
WEED_MYSQL_INTERPOLATEPARAMS: "true"
|
||||
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
WEED_LEVELDB2_ENABLED: "true"
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
|
||||
# directories under this folder will be automatically creating a separate bucket
|
||||
WEED_FILER_BUCKETS_FOLDER: "/buckets"
|
||||
|
||||
|
||||
# secret env variables
|
||||
secretExtraEnvironmentVars: []
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: username
|
||||
# WEED_POSTGRES_PASSWORD:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: password
|
||||
|
||||
s3:
|
||||
enabled: true
|
||||
port: 8333
|
||||
#allow empty folders
|
||||
# add additional https port
|
||||
httpsPort: 0
|
||||
# allow empty folders
|
||||
allowEmptyFolder: false
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
@@ -428,9 +448,11 @@ s3:
|
||||
replicas: 1
|
||||
bindAddress: 0.0.0.0
|
||||
port: 8333
|
||||
# add additional https port
|
||||
httpsPort: 0
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
#allow empty folders
|
||||
# allow empty folders
|
||||
allowEmptyFolder: true
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
@@ -449,7 +471,7 @@ s3:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: null
|
||||
resources: {}
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -459,10 +481,10 @@ s3:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -479,5 +501,5 @@ certificates:
|
||||
ipAddresses: []
|
||||
keyAlgorithm: rsa
|
||||
keySize: 2048
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
@@ -1,62 +0,0 @@
|
||||
# SEAWEEDFS - helm chart (2.x)
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
|
||||
and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
|
||||
with ENV.
|
||||
* cert config exists and can be enabled, but not been tested.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes node have labels which help to define which node(Host) will run which pod:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server & the masters, currently all pods (master/volume/filer)
|
||||
have anti-affinity rule to disallow running multiple pod type on the same host.
|
||||
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
|
||||
please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
on the volume stateful set added support for K8S PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
you can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels.
|
||||
|
||||
most of the configuration are available through values.yaml
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -21,7 +21,7 @@ public class FilerSslContext {
|
||||
String home = System.getProperty("user.home");
|
||||
File f1 = new File("./"+securityFileName);
|
||||
File f2 = new File(home + "/.seaweedfs/"+securityFileName);
|
||||
File f3 = new File(home + "/etc/seaweedfs/"+securityFileName);
|
||||
File f3 = new File("/etc/seaweedfs/"+securityFileName);
|
||||
|
||||
File securityFile = f1.exists()? f1 : f2.exists() ? f2 : f3.exists()? f3 : null;
|
||||
|
||||
|
||||
@@ -68,6 +68,13 @@ service SeaweedFiler {
|
||||
|
||||
rpc CacheRemoteObjectToLocalCluster (CacheRemoteObjectToLocalClusterRequest) returns (CacheRemoteObjectToLocalClusterResponse) {
|
||||
}
|
||||
|
||||
rpc Lock(LockRequest) returns (LockResponse) {
|
||||
}
|
||||
rpc Unlock(UnlockRequest) returns (UnlockResponse) {
|
||||
}
|
||||
rpc TransferLocks(TransferLocksRequest) returns (TransferLocksResponse) {
|
||||
}
|
||||
}
|
||||
|
||||
//////////////////////////////////////////////////
|
||||
@@ -428,3 +435,37 @@ message CacheRemoteObjectToLocalClusterRequest {
|
||||
message CacheRemoteObjectToLocalClusterResponse {
|
||||
Entry entry = 1;
|
||||
}
|
||||
|
||||
/////////////////////////
|
||||
// distributed lock management
|
||||
/////////////////////////
|
||||
message LockRequest {
|
||||
string name = 1;
|
||||
int64 seconds_to_lock = 2;
|
||||
string renew_token = 3;
|
||||
bool is_moved = 4;
|
||||
}
|
||||
message LockResponse {
|
||||
string renew_token = 1;
|
||||
string moved_to = 2;
|
||||
string error = 3;
|
||||
}
|
||||
message UnlockRequest {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
bool is_moved = 3;
|
||||
}
|
||||
message UnlockResponse {
|
||||
string error = 1;
|
||||
string moved_to = 2;
|
||||
}
|
||||
message Lock {
|
||||
string name = 1;
|
||||
string renew_token = 2;
|
||||
int64 expired_at_ns = 3;
|
||||
}
|
||||
message TransferLocksRequest {
|
||||
repeated Lock locks = 1;
|
||||
}
|
||||
message TransferLocksResponse {
|
||||
}
|
||||
@@ -23,7 +23,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>2.10.2</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
@@ -5,6 +5,10 @@
|
||||
<artifactId>copier</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>18</maven.compiler.source>
|
||||
<maven.compiler.target>18</maven.compiler.target>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
@@ -42,6 +44,19 @@ public class HighLevelMultipartUpload {
|
||||
.withS3Client(s3Client)
|
||||
.build();
|
||||
|
||||
|
||||
if (!s3Client.doesBucketExistV2(bucketName)) {
|
||||
// Because the CreateBucketRequest object doesn't specify a region, the
|
||||
// bucket is created in the region specified in the client.
|
||||
s3Client.createBucket(new CreateBucketRequest(bucketName));
|
||||
|
||||
// Verify that the bucket was created by retrieving it and checking its location.
|
||||
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
|
||||
System.out.println("Bucket location: " + bucketLocation);
|
||||
} else {
|
||||
System.out.println("Bucket already exists");
|
||||
}
|
||||
|
||||
// TransferManager processes all transfers asynchronously,
|
||||
// so this call returns immediately.
|
||||
Upload upload = tm.upload(bucketName, keyName, file);
|
||||
|
||||
@@ -2452,7 +2452,27 @@
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "default",
|
||||
"value": "default"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"queryValue": "",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-3h",
|
||||
|
||||
@@ -1895,7 +1895,27 @@
|
||||
"style": "dark",
|
||||
"tags": [],
|
||||
"templating": {
|
||||
"list": []
|
||||
"list": [
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "default",
|
||||
"value": "default"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
"label": "Datasource",
|
||||
"multi": false,
|
||||
"name": "DS_PROMETHEUS",
|
||||
"options": [],
|
||||
"query": "prometheus",
|
||||
"queryValue": "",
|
||||
"refresh": 1,
|
||||
"regex": "",
|
||||
"skipUrlSync": false,
|
||||
"type": "datasource"
|
||||
}
|
||||
]
|
||||
},
|
||||
"time": {
|
||||
"from": "now-5m",
|
||||
|
||||
@@ -2179,8 +2179,8 @@
|
||||
{
|
||||
"current": {
|
||||
"selected": true,
|
||||
"text": "clickhouse-prom",
|
||||
"value": "clickhouse-prom"
|
||||
"text": "default",
|
||||
"value": "default"
|
||||
},
|
||||
"hide": 0,
|
||||
"includeAll": false,
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
|
||||
<properties>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
<guava.version>32.0.0-jre</guava.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/retry"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/credentials"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := MyConfig{
|
||||
Key: "any",
|
||||
Secret: "any",
|
||||
Region: "US",
|
||||
Endpoint: MyEndpoint{
|
||||
URL: "http://localhost:8333",
|
||||
},
|
||||
Bucket: MyBucketConfig{
|
||||
Name: "newbucket",
|
||||
Versioning: false,
|
||||
},
|
||||
MaxBackoffDelay: aws.Int(int(time.Second * 5)),
|
||||
MaxRetryAttempts: aws.Int(1),
|
||||
}
|
||||
|
||||
awsCfg, err := MyAwsConfig(cfg)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
svc := s3.NewFromConfig(*awsCfg, func(o *s3.Options) {
|
||||
o.UsePathStyle = true
|
||||
})
|
||||
|
||||
// Use the S3 client to interact with SeaweedFS
|
||||
// ...
|
||||
// Example: List all buckets
|
||||
result, err := svc.ListBuckets(context.Background(), &s3.ListBucketsInput{})
|
||||
// no errors - got list of buckets
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Print the list of buckets
|
||||
for _, bucket := range result.Buckets {
|
||||
println(*bucket.Name)
|
||||
}
|
||||
|
||||
bucket := "bucket1"
|
||||
_, err = svc.HeadBucket(context.Background(), &s3.HeadBucketInput{Bucket: &bucket})
|
||||
// ERROR HERE
|
||||
if err != nil {
|
||||
println(err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// === helpers
|
||||
|
||||
func MyAwsConfig(cfg MyConfig) (*aws.Config, error) {
|
||||
|
||||
cred := aws.NewCredentialsCache(credentials.NewStaticCredentialsProvider(cfg.Key, cfg.Secret, ""))
|
||||
customResolver := aws.EndpointResolverWithOptionsFunc(
|
||||
func(service, region string, options ...interface{}) (aws.Endpoint, error) {
|
||||
return aws.Endpoint{
|
||||
URL: cfg.Endpoint.URL,
|
||||
SigningRegion: cfg.Region,
|
||||
}, nil
|
||||
})
|
||||
|
||||
awsCfg, err := config.LoadDefaultConfig(context.TODO(),
|
||||
config.WithRegion(cfg.Region),
|
||||
config.WithCredentialsProvider(cred),
|
||||
config.WithEndpointResolverWithOptions(customResolver),
|
||||
config.WithRetryer(func() aws.Retryer {
|
||||
r := retry.AddWithMaxAttempts(retry.NewStandard(), *cfg.MaxRetryAttempts)
|
||||
return retry.AddWithMaxBackoffDelay(r, time.Duration(*cfg.MaxBackoffDelay*1000*1000))
|
||||
}))
|
||||
return &awsCfg, err
|
||||
}
|
||||
|
||||
type MyConfig struct {
|
||||
// Access key of S3 AWS.
|
||||
Key string
|
||||
// Access secret of S3 AWS.
|
||||
Secret string
|
||||
// Region.
|
||||
Region string
|
||||
// AWS endpoint.
|
||||
Endpoint MyEndpoint
|
||||
// Bucket configuration.
|
||||
Bucket MyBucketConfig
|
||||
// File access.
|
||||
FileAccess MyFileAccessType
|
||||
// Maximum backoff delay (ms, default: 20 sec).
|
||||
MaxBackoffDelay *int
|
||||
// Maximum attempts to retry operation on error (default: 5).
|
||||
MaxRetryAttempts *int
|
||||
}
|
||||
|
||||
type MyBucketConfig struct {
|
||||
// Name of bucket
|
||||
Name string
|
||||
// Enable or not versioning
|
||||
Versioning bool
|
||||
}
|
||||
|
||||
type MyEndpoint struct {
|
||||
URL string
|
||||
}
|
||||
|
||||
type MyFileAccessType byte
|
||||
@@ -79,7 +79,19 @@ func startGenerateMetadata() {
|
||||
|
||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: 0,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: 0,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), metadataFollowOption, eachEntryFunc)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -2,9 +2,10 @@ package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||
@@ -32,7 +33,7 @@ func (scanner *VolumeFileScanner4SeeDat) ReadNeedleBody() bool {
|
||||
|
||||
func (scanner *VolumeFileScanner4SeeDat) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error {
|
||||
t := time.Unix(int64(n.AppendAtNs)/int64(time.Second), int64(n.AppendAtNs)%int64(time.Second))
|
||||
glog.V(0).Infof("%d,%s%x offset %d size %d(%s) cookie %x appendedAt %v",
|
||||
glog.V(0).Infof("%d,%s%08x offset %d size %d(%s) cookie %08x appendedAt %v",
|
||||
*volumeId, n.Id, n.Cookie, offset, n.Size, util.BytesToHumanReadable(uint64(n.Size)), n.Cookie, t)
|
||||
return nil
|
||||
}
|
||||
|
||||
+15
-219
@@ -3,7 +3,6 @@ package cluster
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -19,9 +18,6 @@ type FilerGroupName string
|
||||
type DataCenter string
|
||||
type Rack string
|
||||
|
||||
type Leaders struct {
|
||||
leaders [3]pb.ServerAddress
|
||||
}
|
||||
type ClusterNode struct {
|
||||
Address pb.ServerAddress
|
||||
Version string
|
||||
@@ -30,10 +26,7 @@ type ClusterNode struct {
|
||||
DataCenter DataCenter
|
||||
Rack Rack
|
||||
}
|
||||
type GroupMembers struct {
|
||||
members map[pb.ServerAddress]*ClusterNode
|
||||
leaders *Leaders
|
||||
}
|
||||
|
||||
type ClusterNodeGroups struct {
|
||||
groupMembers map[FilerGroupName]*GroupMembers
|
||||
sync.RWMutex
|
||||
@@ -51,50 +44,18 @@ func newClusterNodeGroups() *ClusterNodeGroups {
|
||||
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
|
||||
members, found := g.groupMembers[filerGroup]
|
||||
if !found && createIfNotFound {
|
||||
members = &GroupMembers{
|
||||
members: make(map[pb.ServerAddress]*ClusterNode),
|
||||
leaders: &Leaders{},
|
||||
}
|
||||
members = newGroupMembers()
|
||||
g.groupMembers[filerGroup] = members
|
||||
}
|
||||
return members
|
||||
}
|
||||
|
||||
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
|
||||
if existingNode, found := m.members[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
t := &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
}
|
||||
m.members[address] = t
|
||||
return t
|
||||
}
|
||||
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
|
||||
if existingNode, found := m.members[address]; !found {
|
||||
return false
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(m.members, address)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, true)
|
||||
if t := m.addMember(dataCenter, rack, address, version); t != nil {
|
||||
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
|
||||
return buildClusterNodeUpdateMessage(true, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -106,7 +67,7 @@ func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeTyp
|
||||
return nil
|
||||
}
|
||||
if m.removeMember(address) {
|
||||
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
|
||||
return buildClusterNodeUpdateMessage(false, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -122,24 +83,6 @@ func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []
|
||||
}
|
||||
return
|
||||
}
|
||||
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return false
|
||||
}
|
||||
return m.leaders.isOneLeader(address)
|
||||
}
|
||||
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
|
||||
g.Lock()
|
||||
defer g.Unlock()
|
||||
m := g.getGroupMembers(filerGroup, false)
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
return m.leaders.GetLeaders()
|
||||
}
|
||||
|
||||
func NewCluster() *Cluster {
|
||||
return &Cluster{
|
||||
@@ -166,15 +109,7 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCente
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: true,
|
||||
},
|
||||
},
|
||||
}
|
||||
return buildClusterNodeUpdateMessage(true, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -187,15 +122,7 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
|
||||
case MasterType:
|
||||
return []*master_pb.KeepConnectedResponse{
|
||||
{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
return buildClusterNodeUpdateMessage(false, filerGroup, nodeType, address)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -211,145 +138,14 @@ func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType stri
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
|
||||
case MasterType:
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
|
||||
switch nodeType {
|
||||
case FilerType:
|
||||
return cluster.filerGroups.IsOneLeader(filerGroup, address)
|
||||
case BrokerType:
|
||||
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
|
||||
case MasterType:
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
if isAdd {
|
||||
if m.leaders.addLeaderIfVacant(address) {
|
||||
// has added the address as one leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
FilerGroup: string(filerGroup),
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsLeader: true,
|
||||
IsAdd: true,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
FilerGroup: string(filerGroup),
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsLeader: false,
|
||||
IsAdd: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
} else {
|
||||
if m.leaders.removeLeaderIfExists(address) {
|
||||
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
FilerGroup: string(filerGroup),
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsLeader: true,
|
||||
IsAdd: false,
|
||||
},
|
||||
})
|
||||
|
||||
// pick the freshest one, since it is less likely to go away
|
||||
var shortestDuration int64 = math.MaxInt64
|
||||
now := time.Now()
|
||||
var candidateAddress pb.ServerAddress
|
||||
for _, node := range m.members {
|
||||
if m.leaders.isOneLeader(node.Address) {
|
||||
continue
|
||||
}
|
||||
duration := now.Sub(node.CreatedTs).Nanoseconds()
|
||||
if duration < shortestDuration {
|
||||
shortestDuration = duration
|
||||
candidateAddress = node.Address
|
||||
}
|
||||
}
|
||||
if candidateAddress != "" {
|
||||
m.leaders.addLeaderIfVacant(candidateAddress)
|
||||
// added a new leader
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
NodeType: nodeType,
|
||||
Address: string(candidateAddress),
|
||||
IsLeader: true,
|
||||
IsAdd: true,
|
||||
},
|
||||
})
|
||||
}
|
||||
} else {
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
FilerGroup: string(filerGroup),
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsLeader: false,
|
||||
IsAdd: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (leaders *Leaders) addLeaderIfVacant(address pb.ServerAddress) (hasChanged bool) {
|
||||
if leaders.isOneLeader(address) {
|
||||
return
|
||||
}
|
||||
for i := 0; i < len(leaders.leaders); i++ {
|
||||
if leaders.leaders[i] == "" {
|
||||
leaders.leaders[i] = address
|
||||
hasChanged = true
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
func (leaders *Leaders) removeLeaderIfExists(address pb.ServerAddress) (hasChanged bool) {
|
||||
if !leaders.isOneLeader(address) {
|
||||
return
|
||||
}
|
||||
for i := 0; i < len(leaders.leaders); i++ {
|
||||
if leaders.leaders[i] == address {
|
||||
leaders.leaders[i] = ""
|
||||
hasChanged = true
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
func (leaders *Leaders) isOneLeader(address pb.ServerAddress) bool {
|
||||
for i := 0; i < len(leaders.leaders); i++ {
|
||||
if leaders.leaders[i] == address {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
func (leaders *Leaders) GetLeaders() (addresses []pb.ServerAddress) {
|
||||
for i := 0; i < len(leaders.leaders); i++ {
|
||||
if leaders.leaders[i] != "" {
|
||||
addresses = append(addresses, leaders.leaders[i])
|
||||
}
|
||||
}
|
||||
func buildClusterNodeUpdateMessage(isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
|
||||
result = append(result, &master_pb.KeepConnectedResponse{
|
||||
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
|
||||
FilerGroup: string(filerGroup),
|
||||
NodeType: nodeType,
|
||||
Address: string(address),
|
||||
IsAdd: isAdd,
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -2,52 +2,11 @@ package cluster
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClusterAddRemoveNodes(t *testing.T) {
|
||||
c := NewCluster()
|
||||
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:1"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:2"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:3"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:4"), "23.45")
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:5"), "23.45")
|
||||
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:6"), "23.45")
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:1"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
assert.Equal(t, []pb.ServerAddress{
|
||||
pb.ServerAddress("111:6"),
|
||||
pb.ServerAddress("111:2"),
|
||||
pb.ServerAddress("111:3"),
|
||||
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
|
||||
|
||||
// remove oldest
|
||||
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
|
||||
|
||||
}
|
||||
|
||||
func TestConcurrentAddRemoveNodes(t *testing.T) {
|
||||
c := NewCluster()
|
||||
var wg sync.WaitGroup
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"time"
|
||||
)
|
||||
|
||||
type GroupMembers struct {
|
||||
members map[pb.ServerAddress]*ClusterNode
|
||||
}
|
||||
|
||||
func newGroupMembers() *GroupMembers {
|
||||
return &GroupMembers{
|
||||
members: make(map[pb.ServerAddress]*ClusterNode),
|
||||
}
|
||||
}
|
||||
|
||||
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
|
||||
if existingNode, found := m.members[address]; found {
|
||||
existingNode.counter++
|
||||
return nil
|
||||
}
|
||||
t := &ClusterNode{
|
||||
Address: address,
|
||||
Version: version,
|
||||
counter: 1,
|
||||
CreatedTs: time.Now(),
|
||||
DataCenter: dataCenter,
|
||||
Rack: rack,
|
||||
}
|
||||
m.members[address] = t
|
||||
return t
|
||||
}
|
||||
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
|
||||
if existingNode, found := m.members[address]; !found {
|
||||
return false
|
||||
} else {
|
||||
existingNode.counter--
|
||||
if existingNode.counter <= 0 {
|
||||
delete(m.members, address)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (m *GroupMembers) GetMembers() (addresses []pb.ServerAddress) {
|
||||
for k := range m.members {
|
||||
addresses = append(addresses, k)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,162 @@
|
||||
package cluster
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"google.golang.org/grpc"
|
||||
"time"
|
||||
)
|
||||
|
||||
type LockClient struct {
|
||||
grpcDialOption grpc.DialOption
|
||||
maxLockDuration time.Duration
|
||||
sleepDuration time.Duration
|
||||
}
|
||||
|
||||
func NewLockClient(grpcDialOption grpc.DialOption) *LockClient {
|
||||
return &LockClient{
|
||||
grpcDialOption: grpcDialOption,
|
||||
maxLockDuration: 5 * time.Second,
|
||||
sleepDuration: 4 * time.Millisecond,
|
||||
}
|
||||
}
|
||||
|
||||
type LiveLock struct {
|
||||
key string
|
||||
renewToken string
|
||||
expireAtNs int64
|
||||
filer pb.ServerAddress
|
||||
cancelCh chan struct{}
|
||||
grpcDialOption grpc.DialOption
|
||||
isLocked bool
|
||||
}
|
||||
|
||||
// NewLockWithTimeout locks the key with the given duration
|
||||
func (lc *LockClient) NewLockWithTimeout(filer pb.ServerAddress, key string, lockDuration time.Duration) (lock *LiveLock) {
|
||||
return lc.doNewLock(filer, key, lockDuration)
|
||||
}
|
||||
|
||||
// NewLock creates a lock with a very long duration
|
||||
func (lc *LockClient) NewLock(filer pb.ServerAddress, key string) (lock *LiveLock) {
|
||||
return lc.doNewLock(filer, key, lock_manager.MaxDuration)
|
||||
}
|
||||
|
||||
func (lc *LockClient) doNewLock(filer pb.ServerAddress, key string, lockDuration time.Duration) (lock *LiveLock) {
|
||||
lock = &LiveLock{
|
||||
key: key,
|
||||
filer: filer,
|
||||
cancelCh: make(chan struct{}),
|
||||
expireAtNs: time.Now().Add(lockDuration).UnixNano(),
|
||||
grpcDialOption: lc.grpcDialOption,
|
||||
}
|
||||
var needRenewal bool
|
||||
if lockDuration > lc.maxLockDuration {
|
||||
lockDuration = lc.maxLockDuration
|
||||
needRenewal = true
|
||||
}
|
||||
util.RetryForever("create lock:"+key, func() error {
|
||||
errorMessage, err := lock.doLock(lockDuration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if errorMessage != "" {
|
||||
return fmt.Errorf("%v", errorMessage)
|
||||
}
|
||||
return nil
|
||||
}, func(err error) (shouldContinue bool) {
|
||||
if err != nil {
|
||||
glog.Warningf("create lock %s: %s", key, err)
|
||||
}
|
||||
return lock.renewToken == ""
|
||||
})
|
||||
|
||||
lock.isLocked = true
|
||||
|
||||
if needRenewal {
|
||||
go lc.keepLock(lock)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (lock *LiveLock) IsLocked() bool {
|
||||
return lock.isLocked
|
||||
}
|
||||
|
||||
func (lock *LiveLock) Unlock() error {
|
||||
close(lock.cancelCh)
|
||||
return pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
_, err := client.Unlock(context.Background(), &filer_pb.UnlockRequest{
|
||||
Name: lock.key,
|
||||
RenewToken: lock.renewToken,
|
||||
})
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func (lc *LockClient) keepLock(lock *LiveLock) {
|
||||
for {
|
||||
select {
|
||||
case <-time.After(lc.sleepDuration):
|
||||
// renew the lock if lock.expireAtNs is still greater than now
|
||||
util.RetryForever("keep lock:"+lock.key, func() error {
|
||||
lockDuration := time.Duration(lock.expireAtNs-time.Now().UnixNano()) * time.Nanosecond
|
||||
if lockDuration > lc.maxLockDuration {
|
||||
lockDuration = lc.maxLockDuration
|
||||
}
|
||||
if lockDuration <= 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
errorMessage, err := lock.doLock(lockDuration)
|
||||
if err != nil {
|
||||
lock.isLocked = false
|
||||
return err
|
||||
}
|
||||
if errorMessage != "" {
|
||||
lock.isLocked = false
|
||||
return fmt.Errorf("%v", errorMessage)
|
||||
}
|
||||
return nil
|
||||
}, func(err error) (shouldContinue bool) {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
glog.Warningf("keep lock %s: %v", lock.key, err)
|
||||
return true
|
||||
})
|
||||
if !lock.isLocked {
|
||||
return
|
||||
}
|
||||
case <-lock.cancelCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (lock *LiveLock) doLock(lockDuration time.Duration) (errorMessage string, err error) {
|
||||
err = pb.WithFilerClient(false, 0, lock.filer, lock.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
|
||||
resp, err := client.Lock(context.Background(), &filer_pb.LockRequest{
|
||||
Name: lock.key,
|
||||
SecondsToLock: int64(lockDuration.Seconds()),
|
||||
RenewToken: lock.renewToken,
|
||||
IsMoved: false,
|
||||
})
|
||||
if err == nil {
|
||||
lock.renewToken = resp.RenewToken
|
||||
}
|
||||
if resp != nil {
|
||||
errorMessage = resp.Error
|
||||
if resp.MovedTo != "" {
|
||||
lock.filer = pb.ServerAddress(resp.MovedTo)
|
||||
}
|
||||
}
|
||||
return err
|
||||
})
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
package lock_manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"time"
|
||||
)
|
||||
|
||||
const MaxDuration = 1<<63 - 1
|
||||
|
||||
var NoLockServerError = fmt.Errorf("no lock server found")
|
||||
|
||||
type DistributedLockManager struct {
|
||||
lockManager *LockManager
|
||||
LockRing *LockRing
|
||||
Host pb.ServerAddress
|
||||
}
|
||||
|
||||
func NewDistributedLockManager(host pb.ServerAddress) *DistributedLockManager {
|
||||
return &DistributedLockManager{
|
||||
lockManager: NewLockManager(),
|
||||
LockRing: NewLockRing(time.Second * 5),
|
||||
Host: host,
|
||||
}
|
||||
}
|
||||
|
||||
func (dlm *DistributedLockManager) Lock(key string, token string) (renewToken string, movedTo pb.ServerAddress, err error) {
|
||||
return dlm.LockWithTimeout(key, MaxDuration, token)
|
||||
}
|
||||
|
||||
func (dlm *DistributedLockManager) LockWithTimeout(key string, expiredAtNs int64, token string) (renewToken string, movedTo pb.ServerAddress, err error) {
|
||||
servers := dlm.LockRing.GetSnapshot()
|
||||
if servers == nil {
|
||||
err = NoLockServerError
|
||||
return
|
||||
}
|
||||
|
||||
server := hashKeyToServer(key, servers)
|
||||
if server != dlm.Host {
|
||||
movedTo = server
|
||||
return
|
||||
}
|
||||
renewToken, err = dlm.lockManager.Lock(key, expiredAtNs, token)
|
||||
return
|
||||
}
|
||||
|
||||
func (dlm *DistributedLockManager) Unlock(key string, token string) (movedTo pb.ServerAddress, err error) {
|
||||
servers := dlm.LockRing.GetSnapshot()
|
||||
if servers == nil {
|
||||
err = NoLockServerError
|
||||
return
|
||||
}
|
||||
|
||||
server := hashKeyToServer(key, servers)
|
||||
if server != dlm.Host {
|
||||
movedTo = server
|
||||
return
|
||||
}
|
||||
_, err = dlm.lockManager.Unlock(key, token)
|
||||
return
|
||||
}
|
||||
|
||||
// InsertLock is used to insert a lock to a server unconditionally
|
||||
// It is used when a server is down and the lock is moved to another server
|
||||
func (dlm *DistributedLockManager) InsertLock(key string, expiredAtNs int64, token string) {
|
||||
dlm.lockManager.InsertLock(key, expiredAtNs, token)
|
||||
}
|
||||
func (dlm *DistributedLockManager) SelectNotOwnedLocks(servers []pb.ServerAddress) (locks []*Lock) {
|
||||
return dlm.lockManager.SelectLocks(func(key string) bool {
|
||||
server := hashKeyToServer(key, servers)
|
||||
return server != dlm.Host
|
||||
})
|
||||
}
|
||||
func (dlm *DistributedLockManager) CalculateTargetServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
|
||||
return hashKeyToServer(key, servers)
|
||||
}
|
||||
|
||||
func (dlm *DistributedLockManager) IsLocal(key string) bool {
|
||||
servers := dlm.LockRing.GetSnapshot()
|
||||
if len(servers) <= 1 {
|
||||
return true
|
||||
}
|
||||
return hashKeyToServer(key, servers) == dlm.Host
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package lock_manager
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/google/uuid"
|
||||
"github.com/puzpuzpuz/xsync/v2"
|
||||
"time"
|
||||
)
|
||||
|
||||
var LockErrorNonEmptyTokenOnNewLock = fmt.Errorf("lock: non-empty token on a new lock")
|
||||
var LockErrorNonEmptyTokenOnExpiredLock = fmt.Errorf("lock: non-empty token on an expired lock")
|
||||
var LockErrorTokenMismatch = fmt.Errorf("lock: token mismatch")
|
||||
var UnlockErrorTokenMismatch = fmt.Errorf("unlock: token mismatch")
|
||||
|
||||
// LockManager lock manager
|
||||
type LockManager struct {
|
||||
locks *xsync.MapOf[string, *Lock]
|
||||
}
|
||||
type Lock struct {
|
||||
Token string
|
||||
ExpiredAtNs int64
|
||||
Key string // only used for moving locks
|
||||
}
|
||||
|
||||
func NewLockManager() *LockManager {
|
||||
t := &LockManager{
|
||||
locks: xsync.NewMapOf[*Lock](),
|
||||
}
|
||||
go t.CleanUp()
|
||||
return t
|
||||
}
|
||||
|
||||
func (lm *LockManager) Lock(path string, expiredAtNs int64, token string) (renewToken string, err error) {
|
||||
lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
|
||||
if oldValue != nil {
|
||||
if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < time.Now().UnixNano() {
|
||||
// lock is expired, set to a new lock
|
||||
if token != "" {
|
||||
err = LockErrorNonEmptyTokenOnExpiredLock
|
||||
return nil, false
|
||||
} else {
|
||||
// new lock
|
||||
renewToken = uuid.New().String()
|
||||
return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
|
||||
}
|
||||
}
|
||||
// not expired
|
||||
if oldValue.Token == token {
|
||||
// token matches, renew the lock
|
||||
renewToken = uuid.New().String()
|
||||
return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
|
||||
} else {
|
||||
err = LockErrorTokenMismatch
|
||||
return oldValue, false
|
||||
}
|
||||
} else {
|
||||
if token == "" {
|
||||
// new lock
|
||||
renewToken = uuid.New().String()
|
||||
return &Lock{Token: renewToken, ExpiredAtNs: expiredAtNs}, false
|
||||
} else {
|
||||
err = LockErrorNonEmptyTokenOnNewLock
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (lm *LockManager) Unlock(path string, token string) (isUnlocked bool, err error) {
|
||||
lm.locks.Compute(path, func(oldValue *Lock, loaded bool) (newValue *Lock, delete bool) {
|
||||
if oldValue != nil {
|
||||
now := time.Now()
|
||||
if oldValue.ExpiredAtNs > 0 && oldValue.ExpiredAtNs < now.UnixNano() {
|
||||
// lock is expired, delete it
|
||||
isUnlocked = true
|
||||
return nil, true
|
||||
}
|
||||
if oldValue.Token == token {
|
||||
if oldValue.ExpiredAtNs <= now.UnixNano() {
|
||||
isUnlocked = true
|
||||
return nil, true
|
||||
}
|
||||
return oldValue, false
|
||||
} else {
|
||||
isUnlocked = false
|
||||
err = UnlockErrorTokenMismatch
|
||||
return oldValue, false
|
||||
}
|
||||
} else {
|
||||
isUnlocked = true
|
||||
return nil, true
|
||||
}
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (lm *LockManager) CleanUp() {
|
||||
for {
|
||||
time.Sleep(1 * time.Minute)
|
||||
now := time.Now().UnixNano()
|
||||
lm.locks.Range(func(key string, value *Lock) bool {
|
||||
if value == nil {
|
||||
return true
|
||||
}
|
||||
if now > value.ExpiredAtNs {
|
||||
lm.locks.Delete(key)
|
||||
return true
|
||||
}
|
||||
return true
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// SelectLocks takes out locks by key
|
||||
// if keyFn return true, the lock will be taken out
|
||||
func (lm *LockManager) SelectLocks(selectFn func(key string) bool) (locks []*Lock) {
|
||||
now := time.Now().UnixNano()
|
||||
lm.locks.Range(func(key string, lock *Lock) bool {
|
||||
if now > lock.ExpiredAtNs {
|
||||
lm.locks.Delete(key)
|
||||
return true
|
||||
}
|
||||
if selectFn(key) {
|
||||
lm.locks.Delete(key)
|
||||
lock.Key = key
|
||||
locks = append(locks, lock)
|
||||
}
|
||||
return true
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// InsertLock inserts a lock unconditionally
|
||||
func (lm *LockManager) InsertLock(path string, expiredAtNs int64, token string) {
|
||||
lm.locks.Store(path, &Lock{Token: token, ExpiredAtNs: expiredAtNs})
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package lock_manager
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type LockRingSnapshot struct {
|
||||
servers []pb.ServerAddress
|
||||
ts time.Time
|
||||
}
|
||||
|
||||
type LockRing struct {
|
||||
sync.RWMutex
|
||||
snapshots []*LockRingSnapshot
|
||||
candidateServers map[pb.ServerAddress]struct{}
|
||||
lastUpdateTime time.Time
|
||||
lastCompactTime time.Time
|
||||
snapshotInterval time.Duration
|
||||
onTakeSnapshot func(snapshot []pb.ServerAddress)
|
||||
}
|
||||
|
||||
func NewLockRing(snapshotInterval time.Duration) *LockRing {
|
||||
return &LockRing{
|
||||
snapshotInterval: snapshotInterval,
|
||||
candidateServers: make(map[pb.ServerAddress]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
func (r *LockRing) SetTakeSnapshotCallback(onTakeSnapshot func(snapshot []pb.ServerAddress)) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
r.onTakeSnapshot = onTakeSnapshot
|
||||
}
|
||||
|
||||
// AddServer adds a server to the ring
|
||||
// if the previous snapshot passed the snapshot interval, create a new snapshot
|
||||
func (r *LockRing) AddServer(server pb.ServerAddress) {
|
||||
r.Lock()
|
||||
|
||||
if _, found := r.candidateServers[server]; found {
|
||||
r.Unlock()
|
||||
return
|
||||
}
|
||||
r.lastUpdateTime = time.Now()
|
||||
r.candidateServers[server] = struct{}{}
|
||||
r.Unlock()
|
||||
|
||||
r.takeSnapshotWithDelayedCompaction()
|
||||
}
|
||||
|
||||
func (r *LockRing) RemoveServer(server pb.ServerAddress) {
|
||||
r.Lock()
|
||||
|
||||
if _, found := r.candidateServers[server]; !found {
|
||||
r.Unlock()
|
||||
return
|
||||
}
|
||||
r.lastUpdateTime = time.Now()
|
||||
delete(r.candidateServers, server)
|
||||
r.Unlock()
|
||||
|
||||
r.takeSnapshotWithDelayedCompaction()
|
||||
}
|
||||
|
||||
func (r *LockRing) SetSnapshot(servers []pb.ServerAddress) {
|
||||
|
||||
sort.Slice(servers, func(i, j int) bool {
|
||||
return servers[i] < servers[j]
|
||||
})
|
||||
|
||||
r.lastUpdateTime = time.Now()
|
||||
|
||||
r.addOneSnapshot(servers)
|
||||
|
||||
go func() {
|
||||
<-time.After(r.snapshotInterval)
|
||||
r.compactSnapshots()
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *LockRing) takeSnapshotWithDelayedCompaction() {
|
||||
r.doTakeSnapshot()
|
||||
|
||||
go func() {
|
||||
<-time.After(r.snapshotInterval)
|
||||
r.compactSnapshots()
|
||||
}()
|
||||
}
|
||||
|
||||
func (r *LockRing) doTakeSnapshot() {
|
||||
servers := r.getSortedServers()
|
||||
|
||||
r.addOneSnapshot(servers)
|
||||
}
|
||||
|
||||
func (r *LockRing) addOneSnapshot(servers []pb.ServerAddress) {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
ts := time.Now()
|
||||
t := &LockRingSnapshot{
|
||||
servers: servers,
|
||||
ts: ts,
|
||||
}
|
||||
r.snapshots = append(r.snapshots, t)
|
||||
for i := len(r.snapshots) - 2; i >= 0; i-- {
|
||||
r.snapshots[i+1] = r.snapshots[i]
|
||||
}
|
||||
r.snapshots[0] = t
|
||||
|
||||
if r.onTakeSnapshot != nil {
|
||||
r.onTakeSnapshot(t.servers)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *LockRing) compactSnapshots() {
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
if r.lastCompactTime.After(r.lastUpdateTime) {
|
||||
return
|
||||
}
|
||||
|
||||
ts := time.Now()
|
||||
// remove old snapshots
|
||||
recentSnapshotIndex := 1
|
||||
for ; recentSnapshotIndex < len(r.snapshots); recentSnapshotIndex++ {
|
||||
if ts.Sub(r.snapshots[recentSnapshotIndex].ts) > r.snapshotInterval {
|
||||
break
|
||||
}
|
||||
}
|
||||
// keep the one that has been running for a while
|
||||
if recentSnapshotIndex+1 <= len(r.snapshots) {
|
||||
r.snapshots = r.snapshots[:recentSnapshotIndex+1]
|
||||
}
|
||||
r.lastCompactTime = ts
|
||||
}
|
||||
|
||||
func (r *LockRing) getSortedServers() []pb.ServerAddress {
|
||||
sortedServers := make([]pb.ServerAddress, 0, len(r.candidateServers))
|
||||
for server := range r.candidateServers {
|
||||
sortedServers = append(sortedServers, server)
|
||||
}
|
||||
sort.Slice(sortedServers, func(i, j int) bool {
|
||||
return sortedServers[i] < sortedServers[j]
|
||||
})
|
||||
return sortedServers
|
||||
}
|
||||
|
||||
func (r *LockRing) GetSnapshot() (servers []pb.ServerAddress) {
|
||||
r.RLock()
|
||||
defer r.RUnlock()
|
||||
|
||||
if len(r.snapshots) == 0 {
|
||||
return
|
||||
}
|
||||
return r.snapshots[0].servers
|
||||
}
|
||||
|
||||
func hashKeyToServer(key string, servers []pb.ServerAddress) pb.ServerAddress {
|
||||
if len(servers) == 0 {
|
||||
return ""
|
||||
}
|
||||
x := util.HashStringToLong(key)
|
||||
if x < 0 {
|
||||
x = -x
|
||||
}
|
||||
x = x % int64(len(servers))
|
||||
return servers[x]
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
package lock_manager
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAddServer(t *testing.T) {
|
||||
r := NewLockRing(100 * time.Millisecond)
|
||||
r.AddServer("localhost:8080")
|
||||
assert.Equal(t, 1, len(r.snapshots))
|
||||
r.AddServer("localhost:8081")
|
||||
r.AddServer("localhost:8082")
|
||||
r.AddServer("localhost:8083")
|
||||
r.AddServer("localhost:8084")
|
||||
r.RemoveServer("localhost:8084")
|
||||
r.RemoveServer("localhost:8082")
|
||||
r.RemoveServer("localhost:8080")
|
||||
|
||||
assert.Equal(t, 8, len(r.snapshots))
|
||||
|
||||
time.Sleep(110 * time.Millisecond)
|
||||
|
||||
assert.Equal(t, 2, len(r.snapshots))
|
||||
|
||||
}
|
||||
|
||||
func TestLockRing(t *testing.T) {
|
||||
r := NewLockRing(100 * time.Millisecond)
|
||||
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081"})
|
||||
assert.Equal(t, 1, len(r.snapshots))
|
||||
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082"})
|
||||
assert.Equal(t, 2, len(r.snapshots))
|
||||
time.Sleep(110 * time.Millisecond)
|
||||
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083"})
|
||||
assert.Equal(t, 3, len(r.snapshots))
|
||||
time.Sleep(110 * time.Millisecond)
|
||||
assert.Equal(t, 2, len(r.snapshots))
|
||||
r.SetSnapshot([]pb.ServerAddress{"localhost:8080", "localhost:8081", "localhost:8082", "localhost:8083", "localhost:8084"})
|
||||
assert.Equal(t, 3, len(r.snapshots))
|
||||
}
|
||||
@@ -22,7 +22,6 @@ func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOp
|
||||
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
|
||||
NodeType: FilerType,
|
||||
Address: node.Address,
|
||||
IsLeader: node.IsLeader,
|
||||
IsAdd: true,
|
||||
CreatedAtNs: node.CreatedAtNs,
|
||||
})
|
||||
|
||||
@@ -137,7 +137,7 @@ func runBackup(cmd *Command, args []string) bool {
|
||||
|
||||
if datSize > stats.TailOffset {
|
||||
// remove the old data
|
||||
v.Destroy()
|
||||
v.Destroy(false)
|
||||
// recreate an empty volume
|
||||
v, err = storage.NewVolume(util.ResolvePath(*s.dir), util.ResolvePath(*s.dir), *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0, 0, 0)
|
||||
if err != nil {
|
||||
|
||||
@@ -60,6 +60,7 @@ type FilerOptions struct {
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -89,10 +90,12 @@ func init() {
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
filerS3Options.portHttps = cmdFiler.Flag.Int("s3.port.https", 0, "s3 server https listen port")
|
||||
filerS3Options.portGrpc = cmdFiler.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
|
||||
filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
||||
filerS3Options.dataCenter = cmdFiler.Flag.String("s3.dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
@@ -183,6 +186,11 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
|
||||
if *filerStartWebDav {
|
||||
filerWebDavOptions.filer = &filerAddress
|
||||
|
||||
if *filerWebDavOptions.disk == "" {
|
||||
filerWebDavOptions.disk = f.diskType
|
||||
}
|
||||
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerWebDavOptions.startWebDav()
|
||||
@@ -243,6 +251,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
DiskType: *fo.diskType,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
||||
@@ -138,6 +138,19 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
}()
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "backup_" + dataSink.GetName(),
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startFrom.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
@@ -196,8 +196,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
|
||||
})
|
||||
|
||||
metaBackup.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, metaBackup.clientEpoch,
|
||||
*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "meta_backup",
|
||||
ClientId: metaBackup.clientId,
|
||||
ClientEpoch: metaBackup.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *metaBackup.filerDirectory,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -107,16 +107,28 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
|
||||
untilTsNs = time.Now().Add(-*tailStop).UnixNano()
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, 0, *tailTarget, nil,
|
||||
time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: clientId,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *tailTarget,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: time.Now().Add(-*tailStart).UnixNano(),
|
||||
StopTsNs: untilTsNs,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
}, pb.TrivialOnError)
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -39,8 +39,21 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: option.bucketsDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
}
|
||||
|
||||
func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
@@ -372,6 +385,9 @@ func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
|
||||
func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {
|
||||
|
||||
if mappings, err := filer.ReadMountMappings(option.grpcDialOption, pb.ServerAddress(*option.filerAddress)); err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
return fmt.Errorf("remote storage is not configured in filer server")
|
||||
}
|
||||
return err
|
||||
} else {
|
||||
option.mappings = mappings
|
||||
|
||||
@@ -63,8 +63,21 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: mountedDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
}
|
||||
|
||||
func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
|
||||
@@ -287,8 +287,20 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
|
||||
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, processor.processedTsWatermark)
|
||||
})
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, clientEpoch,
|
||||
sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: clientName,
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: targetFilerSignature,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: sourceFilerOffsetTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.RetryForeverOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,14 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -163,11 +164,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
HeartbeatInterval: *masterOption.heartbeatInterval,
|
||||
ElectionTimeout: *masterOption.electionTimeout,
|
||||
RaftBootstrap: *m.raftBootstrap,
|
||||
RaftBootstrap: *masterOption.raftBootstrap,
|
||||
}
|
||||
var raftServer *weed_server.RaftServer
|
||||
var err error
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
|
||||
glog.Fatalf("NewHashicorpRaftServer: %s", err)
|
||||
}
|
||||
@@ -180,7 +181,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms.SetRaftServer(raftServer)
|
||||
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
||||
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
|
||||
}
|
||||
// starting grpc server
|
||||
@@ -191,7 +192,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
|
||||
master_pb.RegisterSeaweedServer(grpcS, ms)
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
raftServer.TransportManager.Register(grpcS)
|
||||
} else {
|
||||
protobuf.RegisterRaftServer(grpcS, raftServer)
|
||||
@@ -204,7 +205,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
go grpcS.Serve(grpcL)
|
||||
|
||||
timeSleep := 1500 * time.Millisecond
|
||||
if !*m.raftHashicorp {
|
||||
if !*masterOption.raftHashicorp {
|
||||
go func() {
|
||||
time.Sleep(timeSleep)
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
DirectMount: true,
|
||||
DirectMountFlags: 0,
|
||||
//SyncRead: false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
|
||||
//EnableAcl: true,
|
||||
EnableAcl: true,
|
||||
}
|
||||
if *option.nonempty {
|
||||
fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
|
||||
|
||||
+50
-7
@@ -2,8 +2,11 @@ package command
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"google.golang.org/grpc/credentials/tls/certprovider"
|
||||
"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
|
||||
"google.golang.org/grpc/reflection"
|
||||
"net/http"
|
||||
"time"
|
||||
@@ -29,6 +32,7 @@ type S3Options struct {
|
||||
filer *string
|
||||
bindIp *string
|
||||
port *int
|
||||
portHttps *int
|
||||
portGrpc *int
|
||||
config *string
|
||||
domainName *string
|
||||
@@ -40,6 +44,7 @@ type S3Options struct {
|
||||
auditLogConfig *string
|
||||
localFilerSocket *string
|
||||
dataCenter *string
|
||||
certProvider certprovider.Provider
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -47,6 +52,7 @@ func init() {
|
||||
s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address")
|
||||
s3StandaloneOptions.bindIp = cmdS3.Flag.String("ip.bind", "", "ip address to bind to. Default to localhost.")
|
||||
s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port")
|
||||
s3StandaloneOptions.portHttps = cmdS3.Flag.Int("port.https", 0, "s3 server https listen port")
|
||||
s3StandaloneOptions.portGrpc = cmdS3.Flag.Int("port.grpc", 0, "s3 server grpc listen port")
|
||||
s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}")
|
||||
s3StandaloneOptions.dataCenter = cmdS3.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
|
||||
@@ -150,11 +156,18 @@ func runS3(cmd *Command, args []string) bool {
|
||||
|
||||
}
|
||||
|
||||
// GetCertificateWithUpdate Auto refreshing TSL certificate
|
||||
func (S3opt *S3Options) GetCertificateWithUpdate(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
certs, err := S3opt.certProvider.KeyMaterial(context.Background())
|
||||
return &certs.Certs[0], err
|
||||
}
|
||||
|
||||
func (s3opt *S3Options) startS3Server() bool {
|
||||
|
||||
filerAddress := pb.ServerAddress(*s3opt.filer)
|
||||
|
||||
filerBucketsPath := "/buckets"
|
||||
filerGroup := ""
|
||||
|
||||
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
|
||||
|
||||
@@ -169,6 +182,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
|
||||
}
|
||||
filerBucketsPath = resp.DirBuckets
|
||||
filerGroup = resp.FilerGroup
|
||||
metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
|
||||
glog.V(0).Infof("S3 read filer buckets dir: %s", filerBucketsPath)
|
||||
return nil
|
||||
@@ -200,6 +214,7 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
AllowDeleteBucketNotEmpty: *s3opt.allowDeleteBucketNotEmpty,
|
||||
LocalFilerSocket: localFilerSocket,
|
||||
DataCenter: *s3opt.dataCenter,
|
||||
FilerGroup: filerGroup,
|
||||
})
|
||||
if s3ApiServer_err != nil {
|
||||
glog.Fatalf("S3 API Server startup error: %v", s3ApiServer_err)
|
||||
@@ -242,18 +257,46 @@ func (s3opt *S3Options) startS3Server() bool {
|
||||
go grpcS.Serve(grpcL)
|
||||
|
||||
if *s3opt.tlsPrivateKey != "" {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
|
||||
if s3ApiLocalListener != nil {
|
||||
pemfileOptions := pemfile.Options{
|
||||
CertFile: *s3opt.tlsCertificate,
|
||||
KeyFile: *s3opt.tlsPrivateKey,
|
||||
RefreshDuration: security.CredRefreshingInterval,
|
||||
}
|
||||
if s3opt.certProvider, err = pemfile.NewProvider(pemfileOptions); err != nil {
|
||||
glog.Fatalf("pemfile.NewProvider(%v) failed: %v", pemfileOptions, err)
|
||||
}
|
||||
httpS.TLSConfig = &tls.Config{GetCertificate: s3opt.GetCertificateWithUpdate}
|
||||
if *s3opt.portHttps == 0 {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.port)
|
||||
if s3ApiLocalListener != nil {
|
||||
go func() {
|
||||
if err = httpS.ServeTLS(s3ApiLocalListener, "", ""); err != nil {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err = httpS.ServeTLS(s3ApiListener, "", ""); err != nil {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
} else {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at https port %d", util.Version(), *s3opt.portHttps)
|
||||
s3ApiListenerHttps, s3ApiLocalListenerHttps, _ := util.NewIpAndLocalListeners(
|
||||
*s3opt.bindIp, *s3opt.portHttps, time.Duration(10)*time.Second)
|
||||
if s3ApiLocalListenerHttps != nil {
|
||||
go func() {
|
||||
if err = httpS.ServeTLS(s3ApiLocalListenerHttps, "", ""); err != nil {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
go func() {
|
||||
if err = httpS.ServeTLS(s3ApiLocalListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
|
||||
if err = httpS.ServeTLS(s3ApiListenerHttps, "", ""); err != nil {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
if err = httpS.ServeTLS(s3ApiListener, *s3opt.tlsCertificate, *s3opt.tlsPrivateKey); err != nil {
|
||||
glog.Fatalf("S3 API Server Fail to serve: %v", err)
|
||||
}
|
||||
} else {
|
||||
}
|
||||
if *s3opt.tlsPrivateKey == "" || *s3opt.portHttps > 0 {
|
||||
glog.V(0).Infof("Start Seaweed S3 API Server %s at http port %d", util.Version(), *s3opt.port)
|
||||
if s3ApiLocalListener != nil {
|
||||
go func() {
|
||||
|
||||
@@ -108,7 +108,7 @@ connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
|
||||
|
||||
[postgres2]
|
||||
enabled = false
|
||||
@@ -133,7 +133,7 @@ connection_max_open = 100
|
||||
connection_max_lifetime_seconds = 0
|
||||
# if insert/upsert failing, you can disable upsert or update query syntax to match your RDBMS syntax:
|
||||
enableUpsert = true
|
||||
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
||||
upsertQuery = """UPSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)"""
|
||||
|
||||
[cassandra]
|
||||
# CREATE TABLE filemeta (
|
||||
@@ -262,6 +262,9 @@ routeByLatency = false
|
||||
[etcd]
|
||||
enabled = false
|
||||
servers = "localhost:2379"
|
||||
username = ""
|
||||
password = ""
|
||||
key_prefix = "seaweedfs."
|
||||
timeout = "3s"
|
||||
|
||||
[mongodb]
|
||||
@@ -344,4 +347,4 @@ cert_path=""
|
||||
# Set the private key path
|
||||
key_path=""
|
||||
# The name list used to verify the cn name
|
||||
verify_cn=""
|
||||
verify_cn=""
|
||||
@@ -97,6 +97,7 @@ func init() {
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
|
||||
@@ -115,6 +116,7 @@ func init() {
|
||||
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
|
||||
@@ -253,6 +255,9 @@ func runServer(cmd *Command, args []string) bool {
|
||||
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
|
||||
|
||||
if *isStartingFiler {
|
||||
if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
|
||||
filerOptions.diskType = serverOptions.v.diskType
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(1 * time.Second)
|
||||
filerOptions.startFiler()
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -215,7 +214,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
@@ -265,7 +264,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -337,7 +336,7 @@ func extractToFile(buf []byte, filename, target string) error {
|
||||
|
||||
// Write everything to a temp file
|
||||
dir := filepath.Dir(target)
|
||||
new, err := ioutil.TempFile(dir, "weed")
|
||||
new, err := os.CreateTemp(dir, "weed")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ func init() {
|
||||
|
||||
type EtcdStore struct {
|
||||
client *clientv3.Client
|
||||
etcdKeyPrefix string
|
||||
}
|
||||
|
||||
func (store *EtcdStore) GetName() string {
|
||||
@@ -37,15 +38,19 @@ func (store *EtcdStore) Initialize(configuration weed_util.Configuration, prefix
|
||||
servers = "localhost:2379"
|
||||
}
|
||||
|
||||
username := configuration.GetString(prefix + "username")
|
||||
password := configuration.GetString(prefix + "password")
|
||||
store.etcdKeyPrefix = configuration.GetString(prefix + "key_prefix")
|
||||
|
||||
timeout := configuration.GetString(prefix + "timeout")
|
||||
if timeout == "" {
|
||||
timeout = "3s"
|
||||
}
|
||||
|
||||
return store.initialize(servers, timeout)
|
||||
return store.initialize(servers, username, password, timeout)
|
||||
}
|
||||
|
||||
func (store *EtcdStore) initialize(servers string, timeout string) (err error) {
|
||||
func (store *EtcdStore) initialize(servers string, username string, password string, timeout string) (err error) {
|
||||
glog.Infof("filer store etcd: %s", servers)
|
||||
|
||||
to, err := time.ParseDuration(timeout)
|
||||
@@ -55,6 +60,8 @@ func (store *EtcdStore) initialize(servers string, timeout string) (err error) {
|
||||
|
||||
store.client, err = clientv3.New(clientv3.Config{
|
||||
Endpoints: strings.Split(servers, ","),
|
||||
Username: username,
|
||||
Password: password,
|
||||
DialTimeout: to,
|
||||
})
|
||||
if err != nil {
|
||||
@@ -86,7 +93,7 @@ func (store *EtcdStore) InsertEntry(ctx context.Context, entry *filer.Entry) (er
|
||||
meta = weed_util.MaybeGzipData(meta)
|
||||
}
|
||||
|
||||
if _, err := store.client.Put(ctx, string(key), string(meta)); err != nil {
|
||||
if _, err := store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(meta)); err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
@@ -100,7 +107,7 @@ func (store *EtcdStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (er
|
||||
func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPath) (entry *filer.Entry, err error) {
|
||||
key := genKey(fullpath.DirAndName())
|
||||
|
||||
resp, err := store.client.Get(ctx, string(key))
|
||||
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("get %s : %v", fullpath, err)
|
||||
}
|
||||
@@ -123,7 +130,7 @@ func (store *EtcdStore) FindEntry(ctx context.Context, fullpath weed_util.FullPa
|
||||
func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.FullPath) (err error) {
|
||||
key := genKey(fullpath.DirAndName())
|
||||
|
||||
if _, err := store.client.Delete(ctx, string(key)); err != nil {
|
||||
if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(key)); err != nil {
|
||||
return fmt.Errorf("delete %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
@@ -133,7 +140,7 @@ func (store *EtcdStore) DeleteEntry(ctx context.Context, fullpath weed_util.Full
|
||||
func (store *EtcdStore) DeleteFolderChildren(ctx context.Context, fullpath weed_util.FullPath) (err error) {
|
||||
directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
|
||||
|
||||
if _, err := store.client.Delete(ctx, string(directoryPrefix), clientv3.WithPrefix()); err != nil {
|
||||
if _, err := store.client.Delete(ctx, store.etcdKeyPrefix + string(directoryPrefix), clientv3.WithPrefix()); err != nil {
|
||||
return fmt.Errorf("deleteFolderChildren %s : %v", fullpath, err)
|
||||
}
|
||||
|
||||
@@ -151,7 +158,7 @@ func (store *EtcdStore) ListDirectoryEntries(ctx context.Context, dirPath weed_u
|
||||
lastFileStart = genDirectoryKeyPrefix(dirPath, startFileName)
|
||||
}
|
||||
|
||||
resp, err := store.client.Get(ctx, string(lastFileStart),
|
||||
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(lastFileStart),
|
||||
clientv3.WithFromKey(), clientv3.WithLimit(limit+1))
|
||||
if err != nil {
|
||||
return lastFileName, fmt.Errorf("list %s : %v", dirPath, err)
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
|
||||
|
||||
_, err = store.client.Put(ctx, string(key), string(value))
|
||||
_, err = store.client.Put(ctx, store.etcdKeyPrefix + string(key), string(value))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv put: %v", err)
|
||||
@@ -19,7 +19,7 @@ func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (er
|
||||
|
||||
func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
|
||||
|
||||
resp, err := store.client.Get(ctx, string(key))
|
||||
resp, err := store.client.Get(ctx, store.etcdKeyPrefix + string(key))
|
||||
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("kv get: %v", err)
|
||||
@@ -34,7 +34,7 @@ func (store *EtcdStore) KvGet(ctx context.Context, key []byte) (value []byte, er
|
||||
|
||||
func (store *EtcdStore) KvDelete(ctx context.Context, key []byte) (err error) {
|
||||
|
||||
_, err = store.client.Delete(ctx, string(key))
|
||||
_, err = store.client.Delete(ctx, store.etcdKeyPrefix + string(key))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("kv delete: %v", err)
|
||||
|
||||
@@ -10,7 +10,7 @@ func TestStore(t *testing.T) {
|
||||
// to set up local env
|
||||
if false {
|
||||
store := &EtcdStore{}
|
||||
store.initialize("localhost:2379", "3s")
|
||||
store.initialize("localhost:2379", "", "", "3s")
|
||||
store_test.TestFilerStore(t, store)
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,8 @@ type FileChunkSection struct {
|
||||
visibleIntervals *IntervalList[*VisibleInterval]
|
||||
chunkViews *IntervalList[*ChunkView]
|
||||
reader *ChunkReadAt
|
||||
lock sync.Mutex
|
||||
lock sync.RWMutex
|
||||
isPrepared bool
|
||||
}
|
||||
|
||||
func NewFileChunkSection(si SectionIndex) *FileChunkSection {
|
||||
@@ -61,6 +62,19 @@ func removeGarbageChunks(section *FileChunkSection, garbageFileIds map[string]st
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64) {
|
||||
if section.isPrepared {
|
||||
section.reader.fileSize = fileSize
|
||||
return
|
||||
}
|
||||
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
if section.isPrepared {
|
||||
section.reader.fileSize = fileSize
|
||||
return
|
||||
}
|
||||
|
||||
if section.visibleIntervals == nil {
|
||||
section.visibleIntervals = readResolvedChunks(section.chunks, int64(section.sectionIndex)*SectionSize, (int64(section.sectionIndex)+1)*SectionSize)
|
||||
section.chunks, _ = SeparateGarbageChunks(section.visibleIntervals, section.chunks)
|
||||
@@ -76,23 +90,25 @@ func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64)
|
||||
if section.reader == nil {
|
||||
section.reader = NewChunkReaderAtFromClient(group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
|
||||
}
|
||||
|
||||
section.isPrepared = true
|
||||
section.reader.fileSize = fileSize
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) readDataAt(group *ChunkGroup, fileSize int64, buff []byte, offset int64) (n int, tsNs int64, err error) {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
return section.reader.ReadAtWithTime(buff, offset)
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
|
||||
visible := x.Value
|
||||
@@ -108,10 +124,10 @@ func (section *FileChunkSection) DataStartOffset(group *ChunkGroup, offset int64
|
||||
}
|
||||
|
||||
func (section *FileChunkSection) NextStopOffset(group *ChunkGroup, offset int64, fileSize int64) int64 {
|
||||
section.lock.Lock()
|
||||
defer section.lock.Unlock()
|
||||
|
||||
section.setupForRead(group, fileSize)
|
||||
section.lock.RLock()
|
||||
defer section.lock.RUnlock()
|
||||
|
||||
isAfterOffset := false
|
||||
for x := section.visibleIntervals.Front(); x != nil; x = x.Next {
|
||||
|
||||
+41
-5
@@ -3,6 +3,7 @@ package filer
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/cluster/lock_manager"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -48,6 +49,7 @@ type Filer struct {
|
||||
Signature int32
|
||||
FilerConf *FilerConf
|
||||
RemoteStorage *FilerRemoteStorage
|
||||
Dlm *lock_manager.DistributedLockManager
|
||||
}
|
||||
|
||||
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
|
||||
@@ -59,6 +61,7 @@ func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOptio
|
||||
FilerConf: NewFilerConf(),
|
||||
RemoteStorage: NewFilerRemoteStorage(),
|
||||
UniqueFilerId: util.RandomInt32(),
|
||||
Dlm: lock_manager.NewDistributedLockManager(filerHost),
|
||||
}
|
||||
if f.UniqueFilerId < 0 {
|
||||
f.UniqueFilerId = -f.UniqueFilerId
|
||||
@@ -87,17 +90,50 @@ func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*
|
||||
|
||||
glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
|
||||
f.UniqueFilerEpoch++
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, "bootstrap", f.UniqueFilerId, f.UniqueFilerEpoch, "/", nil,
|
||||
0, snapshotTime.UnixNano(), f.Signature, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
}, pb.FatalOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "bootstrap",
|
||||
ClientId: f.UniqueFilerId,
|
||||
ClientEpoch: f.UniqueFilerEpoch,
|
||||
SelfSignature: f.Signature,
|
||||
PathPrefix: "/",
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: snapshotTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*master_pb.ClusterNodeUpdate, startFrom time.Time) {
|
||||
|
||||
var snapshot []pb.ServerAddress
|
||||
for _, node := range existingNodes {
|
||||
address := pb.ServerAddress(node.Address)
|
||||
snapshot = append(snapshot, address)
|
||||
}
|
||||
f.Dlm.LockRing.SetSnapshot(snapshot)
|
||||
glog.V(0).Infof("%s aggregate from peers %+v", self, snapshot)
|
||||
|
||||
f.MetaAggregator = NewMetaAggregator(f, self, f.GrpcDialOption)
|
||||
f.MasterClient.SetOnPeerUpdateFn(f.MetaAggregator.OnPeerUpdate)
|
||||
f.MasterClient.SetOnPeerUpdateFn(func(update *master_pb.ClusterNodeUpdate, startFrom time.Time) {
|
||||
if update.NodeType != cluster.FilerType {
|
||||
return
|
||||
}
|
||||
address := pb.ServerAddress(update.Address)
|
||||
|
||||
if update.IsAdd {
|
||||
f.Dlm.LockRing.AddServer(address)
|
||||
} else {
|
||||
f.Dlm.LockRing.RemoveServer(address)
|
||||
}
|
||||
f.MetaAggregator.OnPeerUpdate(update, startFrom)
|
||||
})
|
||||
|
||||
for _, peerUpdate := range existingNodes {
|
||||
f.MetaAggregator.OnPeerUpdate(peerUpdate, startFrom)
|
||||
|
||||
@@ -125,7 +125,8 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
|
||||
if string(key) == locationPrefix {
|
||||
return true
|
||||
}
|
||||
rules.Put(key, value)
|
||||
key = bytes.Clone(key)
|
||||
_ = rules.Put(key, value)
|
||||
return true
|
||||
})
|
||||
fc.rules = rules
|
||||
|
||||
Loaded 100 of 228 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user