mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-20 11:23:50 +01:00
Compare commits
108
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
422bfaed69 | ||
|
|
b9ab65dba5 | ||
|
|
2b82d0f871 | ||
|
|
8e80ce9476 | ||
|
|
ff649d615d | ||
|
|
3e69a3aac5 | ||
|
|
04190f9951 | ||
|
|
d6ecde8a96 | ||
|
|
ddc5234f85 | ||
|
|
47c4227318 | ||
|
|
3cc7b90414 | ||
|
|
f1f14e28bc | ||
|
|
d54798c3ae | ||
|
|
6bfb91b539 | ||
|
|
bbd461b2c6 | ||
|
|
5db9fcccd4 | ||
|
|
de4545c28b | ||
|
|
5c783c16a1 | ||
|
|
fba05959ef | ||
|
|
8d8347d2f7 | ||
|
|
60c20dce99 | ||
|
|
279c2627f9 | ||
|
|
f90d23916e | ||
|
|
b2134b78d4 | ||
|
|
fa00882e81 | ||
|
|
f0b2650adb | ||
|
|
adb90bd252 | ||
|
|
5b43c4bb98 | ||
|
|
50dc2fe96b | ||
|
|
29d1312625 | ||
|
|
784daea1fa | ||
|
|
264be0d2d4 | ||
|
|
d695119073 | ||
|
|
57ab1f8516 | ||
|
|
dd71f54c6b | ||
|
|
71b33faef0 | ||
|
|
f5854d13df | ||
|
|
0454bb2d88 | ||
|
|
2268d38263 | ||
|
|
b12d9b8598 | ||
|
|
99c05a9c29 | ||
|
|
4d3a485992 | ||
|
|
b1cdea3503 | ||
|
|
2b0cb47a43 | ||
|
|
98bc93cf31 | ||
|
|
66dd114933 | ||
|
|
e17429223e | ||
|
|
bd70683871 | ||
|
|
59706c89fb | ||
|
|
7111c08289 | ||
|
|
c589e9837c | ||
|
|
244385bf0d | ||
|
|
889ecf5c9d | ||
|
|
0cf4f2e891 | ||
|
|
113f3b76f4 | ||
|
|
9ed7922bed | ||
|
|
e52bb3112d | ||
|
|
281f5ef6a5 | ||
|
|
c5b8a0554a | ||
|
|
cef493dd0d | ||
|
|
bd6f2716d3 | ||
|
|
0bf56298d5 | ||
|
|
8241c9a829 | ||
|
|
1f7934a89c | ||
|
|
6cf43f9f58 | ||
|
|
8cf75a152b | ||
|
|
9db15c28ec | ||
|
|
83d15b50ce | ||
|
|
32862b815b | ||
|
|
c5bcfd8942 | ||
|
|
ffbc6615bd | ||
|
|
1976ca9160 | ||
|
|
214b7cd286 | ||
|
|
f457956948 | ||
|
|
4daa791b45 | ||
|
|
db5515eada | ||
|
|
ef2f741823 | ||
|
|
2c4dbaedf2 | ||
|
|
673214574a | ||
|
|
3227e4175e | ||
|
|
439b79016d | ||
|
|
508de8f492 | ||
|
|
f86c0d7751 | ||
|
|
d531fffc89 | ||
|
|
e70cf681fe | ||
|
|
6b7cb33f54 | ||
|
|
a554419e45 | ||
|
|
28c36913b4 | ||
|
|
f05261157f | ||
|
|
e3c48e987b | ||
|
|
472528f6f0 | ||
|
|
cbff1dbd0e | ||
|
|
00fda86880 | ||
|
|
3f22a9db18 | ||
|
|
ca102a0b0b | ||
|
|
9bb0a9e306 | ||
|
|
545d5d0cc3 | ||
|
|
a9aa2d581f | ||
|
|
79af6b41d4 | ||
|
|
896c6b8d1a | ||
|
|
3d7baa30cd | ||
|
|
1134de8cbe | ||
|
|
292e98cc22 | ||
|
|
5869945f16 | ||
|
|
e037c71ec3 | ||
|
|
67b8c2853a | ||
|
|
302155a6fa | ||
|
|
5520b81551 |
No files matched your search
@@ -44,7 +44,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -60,7 +60,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -93,7 +93,7 @@ jobs:
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
- name: Go Release Binaries Large Disk
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -109,7 +109,7 @@ jobs:
|
||||
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
|
||||
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -44,7 +44,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
@@ -45,7 +45,7 @@ jobs:
|
||||
binary_name: weed
|
||||
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_full"
|
||||
- name: Go Release Large Disk Binaries
|
||||
uses: wangyoucao577/go-release-action@074de155de851f31c28201ff48135d4ecea0fb6c # v1.22
|
||||
uses: wangyoucao577/go-release-action@b98909985b9c1fd7b0aaa4c51257a7ba49995781 # v1.22
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
goos: ${{ matrix.goos }}
|
||||
|
||||
@@ -36,7 +36,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
with:
|
||||
buildkitd-flags: "--debug"
|
||||
-
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -37,7 +37,7 @@ jobs:
|
||||
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
|
||||
-
|
||||
name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
|
||||
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
|
||||
-
|
||||
name: Login to Docker Hub
|
||||
if: github.event_name != 'pull_request'
|
||||
|
||||
@@ -11,4 +11,4 @@ jobs:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94
|
||||
uses: actions/dependency-review-action@f46c48ed6d4f1227fb2d9ea62bf6bcbed315589e
|
||||
@@ -24,7 +24,7 @@ jobs:
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -21,7 +21,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go 1.x
|
||||
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
|
||||
uses: actions/setup-go@4d34df0c2316fe8122ab82dc22947d607c0c91f9 # v2
|
||||
with:
|
||||
go-version: ^1.13
|
||||
id: go
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
name: "helm: publish charts"
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- '*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pages: write
|
||||
|
||||
jobs:
|
||||
release:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Publish Helm charts
|
||||
uses: stefanprodan/helm-gh-pages@master
|
||||
with:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
charts_dir: k8s/charts
|
||||
target_dir: helm
|
||||
branch: gh-pages
|
||||
@@ -0,0 +1,51 @@
|
||||
name: "helm: lint and test charts"
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
paths: ['k8s/**']
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
lint-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@v3
|
||||
with:
|
||||
version: v3.10.0
|
||||
|
||||
- uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.9'
|
||||
check-latest: true
|
||||
|
||||
- name: Set up chart-testing
|
||||
uses: helm/chart-testing-action@v2.4.0
|
||||
|
||||
- name: Run chart-testing (list-changed)
|
||||
id: list-changed
|
||||
run: |
|
||||
changed=$(ct list-changed --chart-dirs k8s/charts)
|
||||
if [[ -n "$changed" ]]; then
|
||||
echo "::set-output name=changed::true"
|
||||
fi
|
||||
|
||||
- name: Run chart-testing (lint)
|
||||
run: ct lint --all --validate-maintainers=false --chart-dirs k8s/charts
|
||||
|
||||
- name: Create kind cluster
|
||||
uses: helm/kind-action@v1.5.0
|
||||
|
||||
- name: Run chart-testing (install)
|
||||
run: ct install --all --chart-dirs k8s/charts
|
||||
@@ -8,7 +8,7 @@
|
||||
[](https://github.com/seaweedfs/seaweedfs/wiki)
|
||||
[](https://hub.docker.com/r/chrislusf/seaweedfs/)
|
||||
[](https://search.maven.org/search?q=g:com.github.chrislusf)
|
||||
|
||||
[](https://artifacthub.io/packages/search?repo=seaweedfs)
|
||||
|
||||

|
||||
|
||||
@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
|
||||
-->
|
||||
|
||||
### Gold Sponsors
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.nodion.com)
|
||||
[](https://www.piknik.com)
|
||||
|
||||
---
|
||||
@@ -66,6 +66,7 @@ Table of Contents
|
||||
* [Compared to GlusterFS, Ceph](#compared-to-glusterfs-ceph)
|
||||
* [Compared to GlusterFS](#compared-to-glusterfs)
|
||||
* [Compared to Ceph](#compared-to-ceph)
|
||||
* [Compared to Minio](#compared-to-minio)
|
||||
* [Dev Plan](#dev-plan)
|
||||
* [Installation Guide](#installation-guide)
|
||||
* [Disk Related Topics](#disk-related-topics)
|
||||
|
||||
+2
-1
@@ -62,7 +62,8 @@ k8s: build
|
||||
dev_registry: build
|
||||
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
|
||||
dev_replicate: build
|
||||
dev_replicate:
|
||||
docker build --build-arg TAGS=gocdk --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.go_build .
|
||||
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
|
||||
dev_auditlog: build
|
||||
|
||||
@@ -3,22 +3,23 @@ module github.com/seaweedfs/seaweedfs
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
cloud.google.com/go v0.107.0 // indirect
|
||||
cloud.google.com/go v0.110.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.28.0
|
||||
cloud.google.com/go/storage v1.29.0
|
||||
cloud.google.com/go/storage v1.30.1
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.38.1
|
||||
github.com/aws/aws-sdk-go v1.44.189
|
||||
github.com/aws/aws-sdk-go v1.44.219
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.2.0
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/dustin/go-humanize v1.0.0
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/eapache/go-resiliency v1.3.0 // indirect
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20230111030713-bf00bc1b83b6 // indirect
|
||||
github.com/eapache/queue v1.1.0 // indirect
|
||||
@@ -27,23 +28,23 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fclairamb/ftpserverlib v0.21.0
|
||||
github.com/fsnotify/fsnotify v1.6.0 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
github.com/go-redsync/redsync/v4 v4.7.1
|
||||
github.com/go-redsync/redsync/v4 v4.8.1
|
||||
github.com/go-sql-driver/mysql v1.7.0
|
||||
github.com/go-zookeeper/zk v1.0.3 // indirect
|
||||
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
|
||||
github.com/gocql/gocql v1.3.2
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/golang/protobuf v1.5.2
|
||||
github.com/golang/protobuf v1.5.3
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/btree v1.1.2
|
||||
github.com/google/go-cmp v0.5.9 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.7.1 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
@@ -58,15 +59,15 @@ require (
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.14 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.6
|
||||
github.com/klauspost/reedsolomon v1.11.7
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.14
|
||||
github.com/linxGnu/grocksdb v1.7.15
|
||||
github.com/magiconair/properties v1.8.7 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -78,10 +79,10 @@ require (
|
||||
github.com/pquerna/cachecontrol v0.1.0
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
github.com/prometheus/client_model v0.3.0 // indirect
|
||||
github.com/prometheus/common v0.37.0 // indirect
|
||||
github.com/prometheus/common v0.39.0 // indirect
|
||||
github.com/prometheus/procfs v0.9.0
|
||||
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
github.com/seaweedfs/goexif v1.0.3
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
@@ -89,7 +90,7 @@ require (
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
github.com/spf13/viper v1.15.0
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/stretchr/testify v1.8.2
|
||||
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
|
||||
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
|
||||
github.com/tidwall/gjson v1.14.4
|
||||
@@ -106,75 +107,75 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.7
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
go.mongodb.org/mongo-driver v1.11.3
|
||||
go.opencensus.io v0.24.0 // indirect
|
||||
gocloud.dev v0.28.0
|
||||
gocloud.dev v0.29.0
|
||||
gocloud.dev/pubsub/natspubsub v0.28.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.3.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.5.0
|
||||
golang.org/x/oauth2 v0.2.0 // indirect
|
||||
golang.org/x/sys v0.4.0
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
golang.org/x/tools v0.5.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.29.0
|
||||
golang.org/x/crypto v0.6.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
|
||||
golang.org/x/image v0.6.0
|
||||
golang.org/x/net v0.8.0
|
||||
golang.org/x/oauth2 v0.6.0 // indirect
|
||||
golang.org/x/sys v0.6.0
|
||||
golang.org/x/text v0.8.0 // indirect
|
||||
golang.org/x/tools v0.7.0
|
||||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
|
||||
google.golang.org/api v0.108.0
|
||||
google.golang.org/api v0.114.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
|
||||
google.golang.org/grpc v1.52.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect
|
||||
google.golang.org/grpc v1.53.0
|
||||
google.golang.org/protobuf v1.29.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
modernc.org/cc/v3 v3.40.0 // indirect
|
||||
modernc.org/ccgo/v3 v3.16.13 // indirect
|
||||
modernc.org/libc v1.22.2 // indirect
|
||||
modernc.org/libc v1.22.3 // indirect
|
||||
modernc.org/mathutil v1.5.0 // indirect
|
||||
modernc.org/memory v1.4.0 // indirect
|
||||
modernc.org/memory v1.5.0 // indirect
|
||||
modernc.org/opt v0.1.3 // indirect
|
||||
modernc.org/sqlite v1.20.1
|
||||
modernc.org/sqlite v1.21.0
|
||||
modernc.org/strutil v1.1.3
|
||||
modernc.org/token v1.0.1 // indirect
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.4.0
|
||||
github.com/arangodb/go-driver v1.4.1
|
||||
github.com/arangodb/go-driver v1.5.2
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.11
|
||||
github.com/hanwen/go-fuse/v2 v2.2.0
|
||||
github.com/hashicorp/raft v1.4.0
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/rabbitmq/amqp091-go v1.7.0
|
||||
github.com/schollz/progressbar/v3 v3.13.0
|
||||
github.com/tikv/client-go/v2 v2.0.5
|
||||
github.com/tikv/client-go/v2 v2.0.6
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.6
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
require (
|
||||
cloud.google.com/go/compute v1.14.0 // indirect
|
||||
cloud.google.com/go/compute v1.18.0 // indirect
|
||||
cloud.google.com/go/compute/metadata v0.2.3 // indirect
|
||||
cloud.google.com/go/iam v0.8.0 // indirect
|
||||
cloud.google.com/go/iam v0.12.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
|
||||
github.com/aws/smithy-go v1.13.4 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
|
||||
github.com/aws/smithy-go v1.13.5 // indirect
|
||||
github.com/benbjohnson/clock v1.3.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
@@ -183,18 +184,18 @@ require (
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
github.com/hashicorp/golang-lru v0.5.4 // indirect
|
||||
github.com/hashicorp/golang-lru v0.6.0 // indirect
|
||||
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
|
||||
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.14 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
@@ -208,17 +209,15 @@ require (
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230201112839-2b853bed8125 // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20230228041042-1e9aca94bab6 // indirect
|
||||
github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 // indirect
|
||||
github.com/rivo/uniseg v0.4.3 // indirect
|
||||
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
|
||||
github.com/spf13/pflag v1.0.5 // indirect
|
||||
github.com/stathat/consistent v1.0.0 // indirect
|
||||
github.com/subosito/gotenv v1.4.2 // indirect
|
||||
github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect
|
||||
github.com/tikv/pd v1.1.0-beta.0.20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230202094356-18df271ce57f // indirect
|
||||
github.com/tikv/pd/client v0.0.0-20230301094509-c82b237672a0 // indirect
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
@@ -229,13 +228,13 @@ require (
|
||||
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
|
||||
go.uber.org/atomic v1.10.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/multierr v1.9.0 // indirect
|
||||
go.uber.org/zap v1.24.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/mod v0.9.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/term v0.4.0 // indirect
|
||||
golang.org/x/term v0.6.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
lukechampine.com/uint128 v1.2.0 // indirect
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
# Artifact Hub repository metadata file
|
||||
#
|
||||
# Some settings like the verified publisher flag or the ignored packages won't
|
||||
# be applied until the next time the repository is processed. Please keep in
|
||||
# mind that the repository won't be processed if it has not changed since the
|
||||
# last time it was processed. Depending on the repository kind, this is checked
|
||||
# in a different way. For Helm http based repositories, we consider it has
|
||||
# changed if the `index.yaml` file changes. For git based repositories, it does
|
||||
# when the hash of the last commit in the branch you set up changes. This does
|
||||
# NOT apply to ownership claim operations, which are processed immediately.
|
||||
#
|
||||
|
||||
repositoryID: 5b2f1fe2-20e5-486e-9746-183484642aa2
|
||||
# owners: # (optional, used to claim repository ownership)
|
||||
# - name: username
|
||||
# email: email
|
||||
File renamed without changes.
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.42"
|
||||
version: "3.42"
|
||||
appVersion: "3.45"
|
||||
version: "3.45"
|
||||
@@ -0,0 +1,81 @@
|
||||
# SEAWEEDFS - helm chart (2.x+)
|
||||
|
||||
## Getting Started
|
||||
|
||||
### Add the helm repo
|
||||
|
||||
`helm repo add seaweedfs https://seaweedfs.github.io/seaweedfs/helm`
|
||||
|
||||
### Install the helm chart
|
||||
|
||||
`helm install seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
### (Recommended) Provide `values.yaml`
|
||||
|
||||
`helm install --values=values.yaml seaweedfs seaweedfs/seaweedfs`
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances) and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer with ENV.
|
||||
* cert config exists and can be enabled, but not been tested, requires cert-manager to be installed.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
|
||||
leveldb is the default database this only supports one filer replica.
|
||||
|
||||
To have multiple filers a external datastore is recommened.
|
||||
|
||||
Such as MySQL-compatible database, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb, postgres) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes nodes can have labels which help to define which node(Host) will run which pod:
|
||||
|
||||
Here is an example:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server and the masters, all pods (master/volume/filer)
|
||||
should have anti-affinity rules to disallow running multiple component pods on the same host.
|
||||
|
||||
If you still want to run multiple pods of the same component (master/volume/filer) on the same host please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
On the volume stateful set added support for k8s PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
You can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels if applicable.
|
||||
|
||||
Most of the configuration are available through values.yaml any pull requests to expand functionality or usability are greatly appreciated. Any pull request must pass [chart-testing](https://github.com/helm/chart-testing).
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+7
-1
@@ -97,6 +97,12 @@ spec:
|
||||
value: {{ $value | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if .Values.filer.secretExtraEnvironmentVars }}
|
||||
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
|
||||
- name: {{ $key }}
|
||||
valueFrom: {{ toYaml $value | nindent 16 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
command:
|
||||
- "/bin/sh"
|
||||
- "-ec"
|
||||
@@ -162,7 +168,7 @@ spec:
|
||||
-s3.auditLogConfig=/etc/sw/filer_s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-master={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: seaweedfs-filer-log-volume
|
||||
mountPath: "/logs/"
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+2
-2
@@ -133,8 +133,8 @@ spec:
|
||||
{{- if .Values.master.garbageThreshold }}
|
||||
-garbageThreshold={{ .Values.master.garbageThreshold }} \
|
||||
{{- end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-master.{{ .Release.Namespace }} \
|
||||
-peers={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name : data-{{ .Release.Namespace }}
|
||||
mountPath: /data
|
||||
+1
-1
@@ -105,7 +105,7 @@ spec:
|
||||
{{- if .Values.s3.auditLogConfig }}
|
||||
-auditLogConfig=/etc/sw/s3_auditLogConfig.json \
|
||||
{{- end }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client:{{ .Values.filer.port }}
|
||||
-filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
|
||||
volumeMounts:
|
||||
- name: logs
|
||||
mountPath: "/logs/"
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
+3
-3
@@ -138,9 +138,9 @@ spec:
|
||||
-fileSizeLimitMB={{ .Values.volume.fileSizeLimitMB }} \
|
||||
{{- end }}
|
||||
-minFreeSpacePercent={{ .Values.volume.minFreeSpacePercent }} \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume \
|
||||
-ip=${POD_NAME}.${SEAWEEDFS_FULLNAME}-volume.{{ .Release.Namespace }} \
|
||||
-compactionMBps={{ .Values.volume.compactionMBps }} \
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
-mserver={{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: "{{ .Values.volume.dir }}/"
|
||||
@@ -187,7 +187,7 @@ spec:
|
||||
port: {{ .Values.volume.port }}
|
||||
scheme: HTTP
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 90
|
||||
periodSeconds: 15
|
||||
successThreshold: 1
|
||||
failureThreshold: 100
|
||||
timeoutSeconds: 30
|
||||
@@ -5,7 +5,7 @@ global:
|
||||
repository: ""
|
||||
imageName: chrislusf/seaweedfs
|
||||
imagePullPolicy: IfNotPresent
|
||||
imagePullSecrets: imagepullsecret
|
||||
imagePullSecrets: ""
|
||||
restartPolicy: Always
|
||||
loggingLevel: 1
|
||||
enableSecurity: false
|
||||
@@ -22,8 +22,8 @@ global:
|
||||
replicationPlacment: "001"
|
||||
extraEnvironmentVars:
|
||||
WEED_CLUSTER_DEFAULT: "sw"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client:8888"
|
||||
WEED_CLUSTER_SW_MASTER: "seaweedfs-master.seaweedfs:9333"
|
||||
WEED_CLUSTER_SW_FILER: "seaweedfs-filer-client.seaweedfs:8888"
|
||||
|
||||
image:
|
||||
registry: ""
|
||||
@@ -44,11 +44,11 @@ master:
|
||||
volumePreallocate: false
|
||||
volumeSizeLimitMB: 1000
|
||||
loggingOverrideLevel: null
|
||||
#number of seconds between heartbeats, default 5
|
||||
# number of seconds between heartbeats, default 5
|
||||
pulseSeconds: null
|
||||
#threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
# threshold to vacuum and reclaim spaces, default 0.3 (30%)
|
||||
garbageThreshold: null
|
||||
#Prometheus push interval in seconds, default 15
|
||||
# Prometheus push interval in seconds, default 15
|
||||
metricsIntervalSec: 15
|
||||
# replication type is XYZ:
|
||||
# X number of replica in other data centers
|
||||
@@ -112,10 +112,10 @@ master:
|
||||
# nodeSelector labels for master pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to master pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -241,7 +241,7 @@ volume:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: null
|
||||
resources: {}
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -251,10 +251,10 @@ volume:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-volume: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-volume: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -358,10 +358,10 @@ filer:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -389,7 +389,7 @@ filer:
|
||||
|
||||
# extraEnvVars is a list of extra enviroment variables to set with the stateful set.
|
||||
extraEnvironmentVars:
|
||||
WEED_MYSQL_ENABLED: "true"
|
||||
WEED_MYSQL_ENABLED: "false"
|
||||
WEED_MYSQL_HOSTNAME: "mysql-db-host"
|
||||
WEED_MYSQL_PORT: "3306"
|
||||
WEED_MYSQL_DATABASE: "sw_database"
|
||||
@@ -400,17 +400,29 @@ filer:
|
||||
# enable usage of memsql as filer backend
|
||||
WEED_MYSQL_INTERPOLATEPARAMS: "true"
|
||||
# if you want to use leveldb2, then should enable "enablePVC". or you may lose your data.
|
||||
WEED_LEVELDB2_ENABLED: "false"
|
||||
WEED_LEVELDB2_ENABLED: "true"
|
||||
# with http DELETE, by default the filer would check whether a folder is empty.
|
||||
# recursive_delete will delete all sub folders and files, similar to "rm -Rf"
|
||||
WEED_FILER_OPTIONS_RECURSIVE_DELETE: "false"
|
||||
# directories under this folder will be automatically creating a separate bucket
|
||||
WEED_FILER_BUCKETS_FOLDER: "/buckets"
|
||||
|
||||
|
||||
# secret env variables
|
||||
secretExtraEnvironmentVars: []
|
||||
# WEED_POSTGRES_USERNAME:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: username
|
||||
# WEED_POSTGRES_PASSWORD:
|
||||
# secretKeyRef:
|
||||
# name: postgres-credentials
|
||||
# key: password
|
||||
|
||||
s3:
|
||||
enabled: true
|
||||
port: 8333
|
||||
#allow empty folders
|
||||
# allow empty folders
|
||||
allowEmptyFolder: false
|
||||
# Suffix of the host name, {bucket}.{domainName}
|
||||
domainName: ""
|
||||
@@ -430,7 +442,7 @@ s3:
|
||||
port: 8333
|
||||
metricsPort: 9327
|
||||
loggingOverrideLevel: null
|
||||
#allow empty folders
|
||||
# allow empty folders
|
||||
allowEmptyFolder: true
|
||||
# enable user & permission to s3 (need to inject to all services)
|
||||
enableAuth: false
|
||||
@@ -449,7 +461,7 @@ s3:
|
||||
# should map directly to the value of the resources field for a PodSpec,
|
||||
# formatted as a multi-line string. By default no direct resource request
|
||||
# is made.
|
||||
resources: null
|
||||
resources: {}
|
||||
|
||||
# Toleration Settings for server pods
|
||||
# This should be a multi-line string matching the Toleration array
|
||||
@@ -459,10 +471,10 @@ s3:
|
||||
# nodeSelector labels for server pod assignment, formatted as a muli-line string.
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
|
||||
# Example:
|
||||
# nodeSelector: |
|
||||
# beta.kubernetes.io/arch: amd64
|
||||
nodeSelector: |
|
||||
sw-backend: "true"
|
||||
beta.kubernetes.io/arch: amd64
|
||||
# nodeSelector: |
|
||||
# sw-backend: "true"
|
||||
|
||||
# used to assign priority to server pods
|
||||
# ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
|
||||
@@ -479,5 +491,5 @@ certificates:
|
||||
ipAddresses: []
|
||||
keyAlgorithm: rsa
|
||||
keySize: 2048
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
duration: 2160h # 90d
|
||||
renewBefore: 360h # 15d
|
||||
@@ -1,62 +0,0 @@
|
||||
# SEAWEEDFS - helm chart (2.x)
|
||||
|
||||
## Info:
|
||||
* master/filer/volume are stateful sets with anti-affinity on the hostname,
|
||||
so your deployment will be spread/HA.
|
||||
* chart is using memsql(mysql) as the filer backend to enable HA (multiple filer instances)
|
||||
and backup/HA memsql can provide.
|
||||
* mysql user/password are created in a k8s secret (secret-seaweedfs-db.yaml) and injected to the filer
|
||||
with ENV.
|
||||
* cert config exists and can be enabled, but not been tested.
|
||||
|
||||
## Prerequisites
|
||||
### Database
|
||||
A running MySQL-compatible database is expected by default, as specified in the `values.yaml` at `filer.extraEnvironmentVars`.
|
||||
This database should be pre-configured and initialized by running:
|
||||
```sql
|
||||
CREATE TABLE IF NOT EXISTS `filemeta` (
|
||||
`dirhash` BIGINT NOT NULL COMMENT 'first 64 bits of MD5 hash value of directory field',
|
||||
`name` VARCHAR(766) NOT NULL COMMENT 'directory or file name',
|
||||
`directory` TEXT NOT NULL COMMENT 'full path to parent directory',
|
||||
`meta` LONGBLOB,
|
||||
PRIMARY KEY (`dirhash`, `name`)
|
||||
) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
|
||||
```
|
||||
|
||||
Alternative database can also be configured (e.g. leveldb) following the instructions at `filer.extraEnvironmentVars`.
|
||||
|
||||
### Node Labels
|
||||
Kubernetes node have labels which help to define which node(Host) will run which pod:
|
||||
* s3/filer/master needs the label **sw-backend=true**
|
||||
* volume need the label **sw-volume=true**
|
||||
|
||||
to label a node to be able to run all pod types in k8s:
|
||||
```
|
||||
kubectl label node YOUR_NODE_NAME sw-volume=true,sw-backend=true
|
||||
```
|
||||
|
||||
on production k8s deployment you will want each pod to have a different host,
|
||||
especially the volume server & the masters, currently all pods (master/volume/filer)
|
||||
have anti-affinity rule to disallow running multiple pod type on the same host.
|
||||
if you still want to run multiple pods of the same type (master/volume/filer) on the same host
|
||||
please set/update the corresponding affinity rule in values.yaml to an empty one:
|
||||
|
||||
```affinity: ""```
|
||||
|
||||
## PVC - storage class ###
|
||||
|
||||
on the volume stateful set added support for K8S PVC, currently example
|
||||
with the simple local-path-provisioner from Rancher (comes included with k3d / k3s)
|
||||
https://github.com/rancher/local-path-provisioner
|
||||
|
||||
you can use ANY storage class you like, just update the correct storage-class
|
||||
for your deployment.
|
||||
|
||||
## current instances config (AIO):
|
||||
1 instance for each type (master/filer+s3/volume)
|
||||
|
||||
you can update the replicas count for each node type in values.yaml,
|
||||
need to add more nodes with the corresponding labels.
|
||||
|
||||
most of the configuration are available through values.yaml
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
@@ -5,6 +5,10 @@
|
||||
<artifactId>copier</artifactId>
|
||||
<packaging>jar</packaging>
|
||||
<version>1.0-SNAPSHOT</version>
|
||||
<properties>
|
||||
<maven.compiler.source>18</maven.compiler.source>
|
||||
<maven.compiler.target>18</maven.compiler.target>
|
||||
</properties>
|
||||
<name>copier</name>
|
||||
<url>http://maven.apache.org</url>
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ import com.amazonaws.client.builder.AwsClientBuilder;
|
||||
import com.amazonaws.regions.Regions;
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
||||
import com.amazonaws.services.s3.model.CreateBucketRequest;
|
||||
import com.amazonaws.services.s3.model.GetBucketLocationRequest;
|
||||
import com.amazonaws.services.s3.transfer.TransferManager;
|
||||
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
|
||||
import com.amazonaws.services.s3.transfer.Upload;
|
||||
@@ -42,6 +44,19 @@ public class HighLevelMultipartUpload {
|
||||
.withS3Client(s3Client)
|
||||
.build();
|
||||
|
||||
|
||||
if (!s3Client.doesBucketExistV2(bucketName)) {
|
||||
// Because the CreateBucketRequest object doesn't specify a region, the
|
||||
// bucket is created in the region specified in the client.
|
||||
s3Client.createBucket(new CreateBucketRequest(bucketName));
|
||||
|
||||
// Verify that the bucket was created by retrieving it and checking its location.
|
||||
String bucketLocation = s3Client.getBucketLocation(new GetBucketLocationRequest(bucketName));
|
||||
System.out.println("Bucket location: " + bucketLocation);
|
||||
} else {
|
||||
System.out.println("Bucket already exists");
|
||||
}
|
||||
|
||||
// TransferManager processes all transfers asynchronously,
|
||||
// so this call returns immediately.
|
||||
Upload upload = tm.upload(bucketName, keyName, file);
|
||||
|
||||
@@ -79,7 +79,19 @@ func startGenerateMetadata() {
|
||||
|
||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), "tail", 0, 0, *dir, nil, 0, 0, 0, eachEntryFunc, pb.TrivialOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: 0,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: 0,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithTransportCredentials(insecure.NewCredentials()), metadataFollowOption, eachEntryFunc)
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -60,6 +60,7 @@ type FilerOptions struct {
|
||||
localSocket *string
|
||||
showUIDirectoryDelete *bool
|
||||
downloadMaxMBps *int
|
||||
diskType *string
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -89,6 +90,7 @@ func init() {
|
||||
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
f.downloadMaxMBps = cmdFiler.Flag.Int("downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
f.diskType = cmdFiler.Flag.String("disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
// start s3 on filer
|
||||
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
|
||||
@@ -183,6 +185,11 @@ func runFiler(cmd *Command, args []string) bool {
|
||||
|
||||
if *filerStartWebDav {
|
||||
filerWebDavOptions.filer = &filerAddress
|
||||
|
||||
if *filerWebDavOptions.disk == "" {
|
||||
filerWebDavOptions.disk = f.diskType
|
||||
}
|
||||
|
||||
go func(delay time.Duration) {
|
||||
time.Sleep(delay * time.Second)
|
||||
filerWebDavOptions.startWebDav()
|
||||
@@ -243,6 +250,7 @@ func (fo *FilerOptions) startFiler() {
|
||||
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
|
||||
ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
|
||||
DownloadMaxBytesPs: int64(*fo.downloadMaxMBps) * 1024 * 1024,
|
||||
DiskType: *fo.diskType,
|
||||
})
|
||||
if nfs_err != nil {
|
||||
glog.Fatalf("Filer startup error: %v", nfs_err)
|
||||
|
||||
@@ -138,6 +138,19 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
||||
}()
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, "backup_"+dataSink.GetName(), clientId, clientEpoch, sourcePath, nil, startFrom.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "backup_" + dataSink.GetName(),
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startFrom.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
@@ -196,8 +196,21 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
|
||||
})
|
||||
|
||||
metaBackup.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup", metaBackup.clientId, metaBackup.clientEpoch,
|
||||
*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "meta_backup",
|
||||
ClientId: metaBackup.clientId,
|
||||
ClientEpoch: metaBackup.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *metaBackup.filerDirectory,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: startTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -107,16 +107,28 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
|
||||
untilTsNs = time.Now().Add(-*tailStop).UnixNano()
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail", clientId, 0, *tailTarget, nil,
|
||||
time.Now().Add(-*tailStart).UnixNano(), untilTsNs, 0, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "tail",
|
||||
ClientId: clientId,
|
||||
ClientEpoch: 0,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: *tailTarget,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: time.Now().Add(-*tailStart).UnixNano(),
|
||||
StopTsNs: untilTsNs,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if !shouldPrint(resp) {
|
||||
return nil
|
||||
}, pb.TrivialOnError)
|
||||
}
|
||||
if err := eachEntryFunc(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if tailErr != nil {
|
||||
fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)
|
||||
|
||||
@@ -39,8 +39,21 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: option.bucketsDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
}
|
||||
|
||||
func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
|
||||
@@ -63,8 +63,21 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
||||
lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
|
||||
|
||||
option.clientEpoch++
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync", option.clientId, option.clientEpoch,
|
||||
mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, 0, processEventFnWithOffset, pb.TrivialOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "filer.remote.sync",
|
||||
ClientId: option.clientId,
|
||||
ClientEpoch: option.clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: mountedDir,
|
||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastOffsetTs.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.TrivialOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
}
|
||||
|
||||
func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
|
||||
|
||||
@@ -287,8 +287,20 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
|
||||
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, processor.processedTsWatermark)
|
||||
})
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId, clientEpoch,
|
||||
sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: clientName,
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: targetFilerSignature,
|
||||
PathPrefix: sourcePath,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: sourceFilerOffsetTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.RetryForeverOnError,
|
||||
}
|
||||
|
||||
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -2,13 +2,14 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
|
||||
"golang.org/x/exp/slices"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -163,11 +164,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
RaftResumeState: *masterOption.raftResumeState,
|
||||
HeartbeatInterval: *masterOption.heartbeatInterval,
|
||||
ElectionTimeout: *masterOption.electionTimeout,
|
||||
RaftBootstrap: *m.raftBootstrap,
|
||||
RaftBootstrap: *masterOption.raftBootstrap,
|
||||
}
|
||||
var raftServer *weed_server.RaftServer
|
||||
var err error
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
|
||||
glog.Fatalf("NewHashicorpRaftServer: %s", err)
|
||||
}
|
||||
@@ -180,7 +181,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
ms.SetRaftServer(raftServer)
|
||||
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
|
||||
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
|
||||
}
|
||||
// starting grpc server
|
||||
@@ -191,7 +192,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
}
|
||||
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
|
||||
master_pb.RegisterSeaweedServer(grpcS, ms)
|
||||
if *m.raftHashicorp {
|
||||
if *masterOption.raftHashicorp {
|
||||
raftServer.TransportManager.Register(grpcS)
|
||||
} else {
|
||||
protobuf.RegisterRaftServer(grpcS, raftServer)
|
||||
@@ -204,7 +205,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
go grpcS.Serve(grpcL)
|
||||
|
||||
timeSleep := 1500 * time.Millisecond
|
||||
if !*m.raftHashicorp {
|
||||
if !*masterOption.raftHashicorp {
|
||||
go func() {
|
||||
time.Sleep(timeSleep)
|
||||
|
||||
|
||||
@@ -182,7 +182,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
DirectMount: true,
|
||||
DirectMountFlags: 0,
|
||||
//SyncRead: false, // set to false to enable the FUSE_CAP_ASYNC_READ capability
|
||||
//EnableAcl: true,
|
||||
EnableAcl: true,
|
||||
}
|
||||
if *option.nonempty {
|
||||
fuseMountOptions.Options = append(fuseMountOptions.Options, "nonempty")
|
||||
|
||||
@@ -97,6 +97,7 @@ func init() {
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
|
||||
@@ -115,6 +116,7 @@ func init() {
|
||||
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
|
||||
filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
|
||||
filerOptions.downloadMaxMBps = cmdServer.Flag.Int("filer.downloadMaxMBps", 0, "download max speed for each download request, in MB per second")
|
||||
filerOptions.diskType = cmdServer.Flag.String("filer.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
|
||||
|
||||
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
|
||||
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
|
||||
@@ -253,6 +255,9 @@ func runServer(cmd *Command, args []string) bool {
|
||||
serverWhiteList := util.StringSplit(*serverWhiteListOption, ",")
|
||||
|
||||
if *isStartingFiler {
|
||||
if *filerOptions.diskType == "" && *serverOptions.v.diskType != "" {
|
||||
filerOptions.diskType = serverOptions.v.diskType
|
||||
}
|
||||
go func() {
|
||||
time.Sleep(1 * time.Second)
|
||||
filerOptions.startFiler()
|
||||
|
||||
@@ -11,7 +11,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
@@ -215,7 +214,7 @@ func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (R
|
||||
return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return Release{}, err
|
||||
}
|
||||
@@ -265,7 +264,7 @@ func getGithubData(ctx context.Context, url string) ([]byte, error) {
|
||||
return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
|
||||
}
|
||||
|
||||
buf, err := ioutil.ReadAll(res.Body)
|
||||
buf, err := io.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -337,7 +336,7 @@ func extractToFile(buf []byte, filename, target string) error {
|
||||
|
||||
// Write everything to a temp file
|
||||
dir := filepath.Dir(target)
|
||||
new, err := ioutil.TempFile(dir, "weed")
|
||||
new, err := os.CreateTemp(dir, "weed")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+17
-4
@@ -87,10 +87,23 @@ func (f *Filer) MaybeBootstrapFromPeers(self pb.ServerAddress, existingNodes []*
|
||||
|
||||
glog.V(0).Infof("bootstrap from %v clientId:%d", earliestNode.Address, f.UniqueFilerId)
|
||||
f.UniqueFilerEpoch++
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, "bootstrap", f.UniqueFilerId, f.UniqueFilerEpoch, "/", nil,
|
||||
0, snapshotTime.UnixNano(), f.Signature, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
}, pb.FatalOnError)
|
||||
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "bootstrap",
|
||||
ClientId: f.UniqueFilerId,
|
||||
ClientEpoch: f.UniqueFilerEpoch,
|
||||
SelfSignature: f.Signature,
|
||||
PathPrefix: "/",
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: snapshotTime.UnixNano(),
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
|
||||
err = pb.FollowMetadata(pb.ServerAddress(earliestNode.Address), f.GrpcDialOption, metadataFollowOption, func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
return Replay(f.Store, resp)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -125,7 +125,8 @@ func (fc *FilerConf) DeleteLocationConf(locationPrefix string) {
|
||||
if string(key) == locationPrefix {
|
||||
return true
|
||||
}
|
||||
rules.Put(key, value)
|
||||
key = bytes.Clone(key)
|
||||
_ = rules.Put(key, value)
|
||||
return true
|
||||
})
|
||||
fc.rules = rules
|
||||
|
||||
+15
-22
@@ -300,31 +300,24 @@ func (c *ChunkStreamReader) prepareBufferFor(offset int64) (err error) {
|
||||
if c.bufferOffset <= offset && offset < c.bufferOffset+int64(len(c.buffer)) {
|
||||
return nil
|
||||
}
|
||||
if c.chunkView == nil {
|
||||
return io.EOF
|
||||
}
|
||||
// glog.V(2).Infof("c.chunkView: %v buffer:[%d,%d) offset:%d totalSize:%d", c.chunkView, c.bufferOffset, c.bufferOffset+int64(len(c.buffer)), offset, c.totalSize)
|
||||
|
||||
// fmt.Printf("fetch for offset %d\n", offset)
|
||||
c.chunkView = c.chunkView.Next
|
||||
if c.chunkView == nil {
|
||||
return io.EOF
|
||||
}
|
||||
|
||||
// positioning within the new chunk
|
||||
chunk := c.chunkView.Value
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
} else {
|
||||
for p := c.head; p != nil; p = p.Next {
|
||||
chunk = p.Value
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
// find a possible chunk view
|
||||
p := c.chunkView
|
||||
for p != nil {
|
||||
chunk := p.Value
|
||||
// glog.V(2).Infof("prepareBufferFor check chunk:[%d,%d)", chunk.ViewOffset, chunk.ViewOffset+int64(chunk.ViewSize))
|
||||
if insideChunk(offset, chunk) {
|
||||
if c.isBufferEmpty() || c.bufferOffset != chunk.ViewOffset {
|
||||
c.chunkView = p
|
||||
return c.fetchChunkToBuffer(chunk)
|
||||
}
|
||||
}
|
||||
if offset < c.bufferOffset {
|
||||
p = p.Prev
|
||||
} else {
|
||||
p = p.Next
|
||||
}
|
||||
}
|
||||
|
||||
return io.EOF
|
||||
|
||||
+47
-47
@@ -1,47 +1,47 @@
|
||||
package images
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
|
||||
srcImage, _, err := image.Decode(read)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
return read, err
|
||||
}
|
||||
|
||||
bounds := srcImage.Bounds()
|
||||
if x2 > bounds.Dx() || y2 > bounds.Dy() {
|
||||
read.Seek(0, 0)
|
||||
return read, nil
|
||||
}
|
||||
|
||||
rectangle := image.Rect(x1, y1, x2, y2)
|
||||
dstImage := imaging.Crop(srcImage, rectangle)
|
||||
var buf bytes.Buffer
|
||||
switch ext {
|
||||
case ".jpg", ".jpeg":
|
||||
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".png":
|
||||
if err = png.Encode(&buf, dstImage); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".gif":
|
||||
if err = gif.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
}
|
||||
return bytes.NewReader(buf.Bytes()), err
|
||||
}
|
||||
package images
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"image"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
)
|
||||
|
||||
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
|
||||
srcImage, _, err := image.Decode(read)
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
return read, err
|
||||
}
|
||||
|
||||
bounds := srcImage.Bounds()
|
||||
if x2 > bounds.Dx() || y2 > bounds.Dy() {
|
||||
read.Seek(0, 0)
|
||||
return read, nil
|
||||
}
|
||||
|
||||
rectangle := image.Rect(x1, y1, x2, y2)
|
||||
dstImage := imaging.Crop(srcImage, rectangle)
|
||||
var buf bytes.Buffer
|
||||
switch ext {
|
||||
case ".jpg", ".jpeg":
|
||||
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".png":
|
||||
if err = png.Encode(&buf, dstImage); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
case ".gif":
|
||||
if err = gif.Encode(&buf, dstImage, nil); err != nil {
|
||||
glog.Error(err)
|
||||
}
|
||||
}
|
||||
return bytes.NewReader(buf.Bytes()), err
|
||||
}
|
||||
@@ -53,7 +53,7 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, int64, e
|
||||
return 0, 0, io.EOF
|
||||
}
|
||||
|
||||
if offset+int64(len(buff)) <= int64(len(entry.Content)) {
|
||||
if offset < int64(len(entry.Content)) {
|
||||
totalRead := copy(buff, entry.Content[offset:])
|
||||
glog.V(4).Infof("file handle read cached %s [%d,%d] %d", fileFullPath, offset, offset+int64(totalRead), totalRead)
|
||||
return int64(totalRead), 0, nil
|
||||
|
||||
@@ -58,9 +58,21 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
||||
}
|
||||
|
||||
var clientEpoch int32
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: "mount",
|
||||
ClientId: selfSignature,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: selfSignature,
|
||||
PathPrefix: dir,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: nil,
|
||||
StartTsNs: lastTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
util.RetryForever("followMetaUpdates", func() error {
|
||||
clientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(client, "mount", selfSignature, clientEpoch, dir, nil, &lastTsNs, 0, selfSignature, processEventFn, pb.FatalOnError)
|
||||
return pb.WithFilerClientFollowMetadata(client, metadataFollowOption, processEventFn)
|
||||
}, func(err error) bool {
|
||||
glog.Errorf("follow metadata updates: %v", err)
|
||||
return true
|
||||
|
||||
@@ -3,14 +3,16 @@ package mount
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"os"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
/** Create a directory
|
||||
@@ -54,9 +56,10 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
|
||||
defer wfs.mapPbIdFromFilerToLocal(newEntry)
|
||||
|
||||
request := &filer_pb.CreateEntryRequest{
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
Directory: string(dirFullPath),
|
||||
Entry: newEntry,
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
}
|
||||
|
||||
glog.V(1).Infof("mkdir: %v", request)
|
||||
|
||||
@@ -2,12 +2,14 @@ package mount
|
||||
|
||||
import (
|
||||
"context"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
/*
|
||||
@@ -72,7 +74,8 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
|
||||
HardLinkId: oldEntry.HardLinkId,
|
||||
HardLinkCounter: oldEntry.HardLinkCounter,
|
||||
},
|
||||
Signatures: []int32{wfs.signature},
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
}
|
||||
|
||||
// apply changes to the filer, and also apply to local metaCache
|
||||
|
||||
@@ -3,13 +3,15 @@ package mount
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"os"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/hanwen/go-fuse/v2/fuse"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
)
|
||||
|
||||
/** Create a symbolic link */
|
||||
@@ -42,7 +44,8 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
|
||||
SymlinkTarget: target,
|
||||
},
|
||||
},
|
||||
Signatures: []int32{wfs.signature},
|
||||
Signatures: []int32{wfs.signature},
|
||||
SkipCheckParentDirectory: true,
|
||||
}
|
||||
|
||||
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
+31
-18
@@ -19,22 +19,35 @@ const (
|
||||
RetryForeverOnError
|
||||
)
|
||||
|
||||
// MetadataFollowOption is used to control the behavior of the metadata following
|
||||
// process. Part of it is used as a cursor to resume the following process.
|
||||
type MetadataFollowOption struct {
|
||||
ClientName string
|
||||
ClientId int32
|
||||
ClientEpoch int32
|
||||
SelfSignature int32
|
||||
PathPrefix string
|
||||
AdditionalPathPrefixes []string
|
||||
DirectoriesToWatch []string
|
||||
StartTsNs int64
|
||||
StopTsNs int64
|
||||
EventErrorType EventErrorType
|
||||
}
|
||||
|
||||
type ProcessMetadataFunc func(resp *filer_pb.SubscribeMetadataResponse) error
|
||||
|
||||
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, clientName string, clientId int32, clientEpoch int32,
|
||||
pathPrefix string, additionalPathPrefixes []string, lastTsNs int64, untilTsNs int64, selfSignature int32,
|
||||
processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
|
||||
func FollowMetadata(filerAddress ServerAddress, grpcDialOption grpc.DialOption, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
||||
|
||||
err := WithFilerClient(true, clientId, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(clientName, clientId, clientEpoch, pathPrefix, additionalPathPrefixes, nil, &lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
|
||||
err := WithFilerClient(true, option.SelfSignature, filerAddress, grpcDialOption, makeSubscribeMetadataFunc(option, processEventFn))
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribing filer meta change: %v", err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, clientName string, clientId int32, clientEpoch int32, pathPrefix string, directoriesToWatch []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) error {
|
||||
func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, option *MetadataFollowOption, processEventFn ProcessMetadataFunc) error {
|
||||
|
||||
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(clientName, clientId, clientEpoch, pathPrefix, nil, directoriesToWatch, lastTsNs, untilTsNs, selfSignature, processEventFn, eventErrorType))
|
||||
err := filerClient.WithFilerClient(true, makeSubscribeMetadataFunc(option, processEventFn))
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribing filer meta change: %v", err)
|
||||
}
|
||||
@@ -42,20 +55,20 @@ func WithFilerClientFollowMetadata(filerClient filer_pb.FilerClient, clientName
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeSubscribeMetadataFunc(clientName string, clientId int32, clientEpoch int32, pathPrefix string, additionalPathPrefixes []string, directoriesToWatch []string, lastTsNs *int64, untilTsNs int64, selfSignature int32, processEventFn ProcessMetadataFunc, eventErrorType EventErrorType) func(client filer_pb.SeaweedFilerClient) error {
|
||||
func makeSubscribeMetadataFunc(option *MetadataFollowOption, processEventFn ProcessMetadataFunc) func(client filer_pb.SeaweedFilerClient) error {
|
||||
return func(client filer_pb.SeaweedFilerClient) error {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
stream, err := client.SubscribeMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
|
||||
ClientName: clientName,
|
||||
PathPrefix: pathPrefix,
|
||||
PathPrefixes: additionalPathPrefixes,
|
||||
Directories: directoriesToWatch,
|
||||
SinceNs: *lastTsNs,
|
||||
Signature: selfSignature,
|
||||
ClientId: clientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
UntilNs: untilTsNs,
|
||||
ClientName: option.ClientName,
|
||||
PathPrefix: option.PathPrefix,
|
||||
PathPrefixes: option.AdditionalPathPrefixes,
|
||||
Directories: option.DirectoriesToWatch,
|
||||
SinceNs: option.StartTsNs,
|
||||
Signature: option.SelfSignature,
|
||||
ClientId: option.ClientId,
|
||||
ClientEpoch: option.ClientEpoch,
|
||||
UntilNs: option.StopTsNs,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("subscribe: %v", err)
|
||||
@@ -71,7 +84,7 @@ func makeSubscribeMetadataFunc(clientName string, clientId int32, clientEpoch in
|
||||
}
|
||||
|
||||
if err := processEventFn(resp); err != nil {
|
||||
switch eventErrorType {
|
||||
switch option.EventErrorType {
|
||||
case TrivialOnError:
|
||||
glog.Errorf("process %v: %v", resp, err)
|
||||
case FatalOnError:
|
||||
@@ -87,7 +100,7 @@ func makeSubscribeMetadataFunc(clientName string, clientId int32, clientEpoch in
|
||||
glog.Errorf("process %v: %v", resp, err)
|
||||
}
|
||||
}
|
||||
*lastTsNs = resp.TsNs
|
||||
option.StartTsNs = resp.TsNs
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,9 +34,21 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, p
|
||||
}
|
||||
|
||||
var clientEpoch int32
|
||||
metadataFollowOption := &pb.MetadataFollowOption{
|
||||
ClientName: clientName,
|
||||
ClientId: s3a.randomClientId,
|
||||
ClientEpoch: clientEpoch,
|
||||
SelfSignature: 0,
|
||||
PathPrefix: prefix,
|
||||
AdditionalPathPrefixes: nil,
|
||||
DirectoriesToWatch: directoriesToWatch,
|
||||
StartTsNs: lastTsNs,
|
||||
StopTsNs: 0,
|
||||
EventErrorType: pb.FatalOnError,
|
||||
}
|
||||
util.RetryForever("followIamChanges", func() error {
|
||||
clientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
|
||||
return pb.WithFilerClientFollowMetadata(s3a, metadataFollowOption, processEventFn)
|
||||
}, func(err error) bool {
|
||||
glog.V(0).Infof("iam follow metadata changes: %v", err)
|
||||
return true
|
||||
@@ -65,7 +77,7 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
|
||||
return nil
|
||||
}
|
||||
|
||||
//reload bucket metadata
|
||||
// reload bucket metadata
|
||||
func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
|
||||
if dir == s3a.option.BucketsPath {
|
||||
if newEntry != nil {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
package s3_constants
|
||||
|
||||
//Amazon S3 predefined groups
|
||||
// Amazon S3 predefined groups
|
||||
var (
|
||||
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
|
||||
|
||||
@@ -46,6 +46,7 @@ func (s3a *S3ApiServer) CopyObjectHandler(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
entry.Extended, err = processMetadataBytes(r.Header, entry.Extended, replaceMeta, replaceTagging)
|
||||
entry.Attributes.Mtime = time.Now().Unix()
|
||||
if err != nil {
|
||||
glog.Errorf("CopyObjectHandler ValidateTags error %s: %v", r.URL, err)
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidTag)
|
||||
|
||||
@@ -146,57 +146,65 @@ func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, m
|
||||
|
||||
// check filer
|
||||
err = s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
|
||||
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
|
||||
if entry.IsDirectory {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
||||
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
|
||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
||||
})
|
||||
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
|
||||
cursor.maxKeys--
|
||||
} else if entry.IsDirectoryKeyObject() {
|
||||
for {
|
||||
empty := true
|
||||
nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, cursor, marker, delimiter, false, func(dir string, entry *filer_pb.Entry) {
|
||||
empty = false
|
||||
if entry.IsDirectory {
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
|
||||
if delimiter == "/" { // A response can contain CommonPrefixes only if you specify a delimiter.
|
||||
commonPrefixes = append(commonPrefixes, PrefixEntry{
|
||||
Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
||||
})
|
||||
//All of the keys (up to 1,000) rolled up into a common prefix count as a single return when calculating the number of returns.
|
||||
cursor.maxKeys--
|
||||
} else if entry.IsDirectoryKeyObject() {
|
||||
contents = append(contents, ListEntry{
|
||||
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||
Owner: CanonicalUser{
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
},
|
||||
StorageClass: "STANDARD",
|
||||
})
|
||||
cursor.maxKeys--
|
||||
}
|
||||
} else {
|
||||
storageClass := "STANDARD"
|
||||
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
storageClass = string(v)
|
||||
}
|
||||
contents = append(contents, ListEntry{
|
||||
Key: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
|
||||
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||
Size: int64(filer.FileSize(entry)),
|
||||
Owner: CanonicalUser{
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
},
|
||||
StorageClass: "STANDARD",
|
||||
StorageClass: StorageClass(storageClass),
|
||||
})
|
||||
cursor.maxKeys--
|
||||
}
|
||||
} else {
|
||||
storageClass := "STANDARD"
|
||||
if v, ok := entry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
storageClass = string(v)
|
||||
}
|
||||
contents = append(contents, ListEntry{
|
||||
Key: fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
|
||||
LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
|
||||
ETag: "\"" + filer.ETag(entry) + "\"",
|
||||
Size: int64(filer.FileSize(entry)),
|
||||
Owner: CanonicalUser{
|
||||
ID: fmt.Sprintf("%x", entry.Attributes.Uid),
|
||||
DisplayName: entry.Attributes.UserName,
|
||||
},
|
||||
StorageClass: StorageClass(storageClass),
|
||||
})
|
||||
cursor.maxKeys--
|
||||
})
|
||||
if doErr != nil {
|
||||
return doErr
|
||||
}
|
||||
})
|
||||
if doErr != nil {
|
||||
return doErr
|
||||
}
|
||||
|
||||
if !cursor.isTruncated {
|
||||
nextMarker = ""
|
||||
} else {
|
||||
if requestDir != "" {
|
||||
nextMarker = requestDir + "/" + nextMarker
|
||||
if cursor.isTruncated {
|
||||
if requestDir != "" {
|
||||
nextMarker = requestDir + "/" + nextMarker
|
||||
}
|
||||
break
|
||||
} else if empty {
|
||||
nextMarker = ""
|
||||
break
|
||||
} else {
|
||||
// start next loop
|
||||
marker = nextMarker
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"google.golang.org/grpc/credentials/insecure"
|
||||
"google.golang.org/grpc/credentials/tls/certprovider/pemfile"
|
||||
"google.golang.org/grpc/security/advancedtls"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -136,7 +136,7 @@ func LoadClientTLS(config *util.ViperProxy, component string) grpc.DialOption {
|
||||
}
|
||||
|
||||
func LoadClientTLSHTTP(clientCertFile string) *tls.Config {
|
||||
clientCerts, err := ioutil.ReadFile(clientCertFile)
|
||||
clientCerts, err := os.ReadFile(clientCertFile)
|
||||
if err != nil {
|
||||
glog.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -295,6 +295,10 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr
|
||||
|
||||
func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) {
|
||||
|
||||
if req.DiskType == "" {
|
||||
req.DiskType = fs.option.DiskType
|
||||
}
|
||||
|
||||
so, err := fs.detectStorageOption(req.Path, req.Collection, req.Replication, req.TtlSec, req.DiskType, req.DataCenter, req.Rack, req.DataNode)
|
||||
if err != nil {
|
||||
glog.V(3).Infof("AssignVolume: %v", err)
|
||||
|
||||
@@ -154,10 +154,11 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
|
||||
}
|
||||
|
||||
chunks = append(chunks, &filer_pb.FileChunk{
|
||||
|
||||
FileId: assignResult.Fid,
|
||||
Offset: localOffset,
|
||||
Size: uint64(size),
|
||||
ModifiedTsNs: time.Now().Unix(),
|
||||
ModifiedTsNs: time.Now().UnixNano(),
|
||||
ETag: etag,
|
||||
Fid: &filer_pb.FileId{
|
||||
VolumeId: uint32(fileId.VolumeId),
|
||||
|
||||
@@ -69,6 +69,7 @@ type FilerOption struct {
|
||||
ConcurrentUploadLimit int64
|
||||
ShowUIDirectoryDelete bool
|
||||
DownloadMaxBytesPs int64
|
||||
DiskType string
|
||||
}
|
||||
|
||||
type FilerServer struct {
|
||||
@@ -194,9 +195,6 @@ func (fs *FilerServer) checkWithMaster() {
|
||||
return fmt.Errorf("get master %s configuration: %v", master, err)
|
||||
}
|
||||
fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
|
||||
if fs.option.DefaultReplication == "" {
|
||||
fs.option.DefaultReplication = resp.DefaultReplication
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if readErr == nil {
|
||||
|
||||
@@ -27,7 +27,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
limit, limit_err := strconv.Atoi(r.FormValue("limit"))
|
||||
if limit_err != nil {
|
||||
limit = 100
|
||||
limit = fs.option.DirListingLimit
|
||||
}
|
||||
|
||||
lastFileName := r.FormValue("lastFileName")
|
||||
|
||||
@@ -4,12 +4,13 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -97,6 +98,11 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request, conte
|
||||
return
|
||||
}
|
||||
|
||||
// When DiskType is empty,use filer's -disk
|
||||
if so.DiskType == "" {
|
||||
so.DiskType = fs.option.DiskType
|
||||
}
|
||||
|
||||
if query.Has("mv.from") {
|
||||
fs.move(ctx, w, r, so)
|
||||
} else {
|
||||
|
||||
@@ -49,7 +49,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
|
||||
if err != nil {
|
||||
if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
|
||||
writeJsonError(w, r, 499, err)
|
||||
} else if strings.HasSuffix(err.Error(), "is a file") {
|
||||
} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
|
||||
writeJsonError(w, r, http.StatusConflict, err)
|
||||
} else {
|
||||
writeJsonError(w, r, http.StatusInternalServerError, err)
|
||||
|
||||
@@ -309,12 +309,13 @@ func (ms *MasterServer) VolumeMarkReadonly(ctx context.Context, req *master_pb.V
|
||||
replicaPlacement, _ := super_block.NewReplicaPlacementFromByte(byte(req.ReplicaPlacement))
|
||||
vl := ms.Topo.GetVolumeLayout(req.Collection, replicaPlacement, needle.LoadTTLFromUint32(req.Ttl), types.ToDiskType(req.DiskType))
|
||||
dataNodes := ms.Topo.Lookup(req.Collection, needle.VolumeId(req.VolumeId))
|
||||
|
||||
for _, dn := range dataNodes {
|
||||
if dn.Ip == req.Ip && dn.Port == int(req.Port) {
|
||||
if req.IsReadonly {
|
||||
vl.SetVolumeUnavailable(dn, needle.VolumeId(req.VolumeId))
|
||||
vl.SetVolumeReadOnly(dn, needle.VolumeId(req.VolumeId))
|
||||
} else {
|
||||
vl.SetVolumeAvailable(dn, needle.VolumeId(req.VolumeId), false)
|
||||
vl.SetVolumeWritable(dn, needle.VolumeId(req.VolumeId))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -316,7 +316,7 @@ func processEachCmd(reg *regexp.Regexp, line string, commandEnv *shell.CommandEn
|
||||
for i := range args {
|
||||
args[i] = strings.Trim(string(cmds[1+i]), "\"'")
|
||||
}
|
||||
cmd := strings.ToLower(cmds[0])
|
||||
cmd := cmds[0]
|
||||
|
||||
for _, c := range shell.Commands {
|
||||
if c.Name() == cmd {
|
||||
|
||||
@@ -121,6 +121,10 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
c.LogLevel = "Error"
|
||||
}
|
||||
|
||||
if err := raft.ValidateConfig(c); err != nil {
|
||||
return nil, fmt.Errorf(`raft.ValidateConfig: %v`, err)
|
||||
}
|
||||
|
||||
if option.RaftBootstrap {
|
||||
os.RemoveAll(path.Join(s.dataDir, ldbFile))
|
||||
os.RemoveAll(path.Join(s.dataDir, sdbFile))
|
||||
|
||||
@@ -4,7 +4,6 @@ import (
|
||||
"encoding/json"
|
||||
transport "github.com/Jille/raft-grpc-transport"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
"os"
|
||||
"path"
|
||||
@@ -90,7 +89,7 @@ func (s *StateMachine) Snapshot() (hashicorpRaft.FSMSnapshot, error) {
|
||||
}
|
||||
|
||||
func (s *StateMachine) Restore(r io.ReadCloser) error {
|
||||
b, err := ioutil.ReadAll(r)
|
||||
b, err := io.ReadAll(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -12,9 +12,11 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"golang.org/x/exp/slices"
|
||||
"io"
|
||||
"math"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -24,10 +26,14 @@ func init() {
|
||||
|
||||
type commandFsVerify struct {
|
||||
env *CommandEnv
|
||||
volumeServers []pb.ServerAddress
|
||||
volumeIds map[uint32][]pb.ServerAddress
|
||||
verbose *bool
|
||||
concurrency *int
|
||||
modifyTimeAgoAtSec int64
|
||||
writer io.Writer
|
||||
waitChan map[string]chan struct{}
|
||||
waitChanLock sync.RWMutex
|
||||
}
|
||||
|
||||
func (c *commandFsVerify) Name() string {
|
||||
@@ -45,10 +51,10 @@ func (c *commandFsVerify) Help() string {
|
||||
func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
|
||||
c.env = commandEnv
|
||||
c.writer = writer
|
||||
|
||||
fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
|
||||
c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
|
||||
modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
|
||||
c.concurrency = fsVerifyCommand.Int("concurrency", 0, "number of parallel verification per volume server")
|
||||
|
||||
if err = fsVerifyCommand.Parse(args); err != nil {
|
||||
return err
|
||||
@@ -60,13 +66,29 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
|
||||
}
|
||||
|
||||
c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
|
||||
c.volumeIds = make(map[uint32][]pb.ServerAddress)
|
||||
c.waitChan = make(map[string]chan struct{})
|
||||
c.volumeServers = []pb.ServerAddress{}
|
||||
defer func() {
|
||||
c.modifyTimeAgoAtSec = 0
|
||||
c.volumeIds = nil
|
||||
c.waitChan = nil
|
||||
c.volumeServers = nil
|
||||
}()
|
||||
|
||||
if err := c.collectVolumeIds(); err != nil {
|
||||
return parseErr
|
||||
}
|
||||
|
||||
fCount, eConut, terr := c.verifyTraverseBfs(path)
|
||||
if *c.concurrency > 0 {
|
||||
for _, volumeServer := range c.volumeServers {
|
||||
volumeServerStr := string(volumeServer)
|
||||
c.waitChan[volumeServerStr] = make(chan struct{}, *c.concurrency)
|
||||
defer close(c.waitChan[volumeServerStr])
|
||||
}
|
||||
}
|
||||
|
||||
fCount, eConut, terr := c.verifyTraverseBfs(path)
|
||||
if terr == nil {
|
||||
fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut)
|
||||
}
|
||||
@@ -76,7 +98,6 @@ func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Wr
|
||||
}
|
||||
|
||||
func (c *commandFsVerify) collectVolumeIds() error {
|
||||
c.volumeIds = make(map[uint32][]pb.ServerAddress)
|
||||
topologyInfo, _, err := collectTopologyInfo(c.env, 0)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -84,15 +105,19 @@ func (c *commandFsVerify) collectVolumeIds() error {
|
||||
eachDataNode(topologyInfo, func(dc string, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
|
||||
for _, diskInfo := range nodeInfo.DiskInfos {
|
||||
for _, vi := range diskInfo.VolumeInfos {
|
||||
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], pb.NewServerAddressFromDataNode(nodeInfo))
|
||||
volumeServer := pb.NewServerAddressFromDataNode(nodeInfo)
|
||||
c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], volumeServer)
|
||||
if !slices.Contains(c.volumeServers, volumeServer) {
|
||||
c.volumeServers = append(c.volumeServers, volumeServer)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.ServerAddress) error {
|
||||
err := operation.WithVolumeServerClient(false, *volumeServer, c.env.option.GrpcDialOption,
|
||||
func (c *commandFsVerify) verifyEntry(volumeServer pb.ServerAddress, fileId *filer_pb.FileId) error {
|
||||
err := operation.WithVolumeServerClient(false, volumeServer, c.env.option.GrpcDialOption,
|
||||
func(client volume_server_pb.VolumeServerClient) error {
|
||||
_, err := client.VolumeNeedleStatus(context.Background(),
|
||||
&volume_server_pb.VolumeNeedleStatusRequest{
|
||||
@@ -104,9 +129,6 @@ func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.
|
||||
if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
|
||||
return err
|
||||
}
|
||||
if *c.verbose {
|
||||
fmt.Fprintf(c.writer, ".")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -140,27 +162,50 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
|
||||
func(outputChan chan interface{}) {
|
||||
for itemEntry := range outputChan {
|
||||
i := itemEntry.(*ItemEntry)
|
||||
fileMsg := fmt.Sprintf("file:%s needle status ", i.path)
|
||||
if *c.verbose {
|
||||
fmt.Fprintf(c.writer, fileMsg)
|
||||
fileMsg = ""
|
||||
}
|
||||
itemPath := string(i.path)
|
||||
fileMsg := fmt.Sprintf("file:%s", itemPath)
|
||||
errItem := make(map[string]error)
|
||||
errItemLock := sync.RWMutex{}
|
||||
for _, chunk := range i.chunks {
|
||||
if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
|
||||
for _, volumeServer := range volumeIds {
|
||||
if err = c.verifyEntry(chunk.Fid, &volumeServer); err != nil {
|
||||
fmt.Fprintf(c.writer, "%sfailed verify %d:%d: %+v\n",
|
||||
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
|
||||
break
|
||||
if *c.concurrency == 0 {
|
||||
if err = c.verifyEntry(volumeServer, chunk.Fid); err != nil {
|
||||
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
|
||||
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
c.waitChanLock.RLock()
|
||||
waitChan, ok := c.waitChan[string(volumeServer)]
|
||||
c.waitChanLock.RUnlock()
|
||||
if !ok {
|
||||
fmt.Fprintf(c.writer, "%s failed to get channel for %s chunk: %d:%d: %+v\n",
|
||||
string(volumeServer), fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
|
||||
continue
|
||||
}
|
||||
waitChan <- struct{}{}
|
||||
go func(fId *filer_pb.FileId, path string, volumeServer pb.ServerAddress, msg string) {
|
||||
if err = c.verifyEntry(volumeServer, fId); err != nil {
|
||||
errItemLock.Lock()
|
||||
errItem[path] = err
|
||||
fmt.Fprintf(c.writer, "%s failed verify needle %d:%d: %+v\n",
|
||||
msg, fId.VolumeId, fId.FileKey, err)
|
||||
errItemLock.Unlock()
|
||||
}
|
||||
<-waitChan
|
||||
}(chunk.Fid, itemPath, volumeServer, fileMsg)
|
||||
}
|
||||
} else {
|
||||
err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
|
||||
fmt.Fprintf(c.writer, "%sfailed verify chunk %d:%d: %+v\n",
|
||||
fmt.Fprintf(c.writer, "%s %d:%d: %+v\n",
|
||||
fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
|
||||
break
|
||||
}
|
||||
}
|
||||
errItemLock.RLock()
|
||||
err, _ = errItem[itemPath]
|
||||
errItemLock.RUnlock()
|
||||
|
||||
if err != nil {
|
||||
errCount++
|
||||
@@ -168,7 +213,7 @@ func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCo
|
||||
}
|
||||
|
||||
if *c.verbose {
|
||||
fmt.Fprintf(c.writer, " verifed\n")
|
||||
fmt.Fprintf(c.writer, "%s needles:%d verifed\n", fileMsg, len(i.chunks))
|
||||
}
|
||||
fileCount++
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
|
||||
collectionSize = collectionInfo.Size
|
||||
fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
|
||||
}
|
||||
fmt.Fprintf(writer, " %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
|
||||
fmt.Fprintf(writer, " %s\tsize:%.0f\tchunk:%.0f", entry.Name, collectionSize, fileCount)
|
||||
if entry.Quota > 0 {
|
||||
fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
|
||||
}
|
||||
|
||||
@@ -330,7 +330,7 @@ func maybeMoveOneVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][]*Vol
|
||||
}
|
||||
|
||||
if candidateVolume.RemoteStorageName != "" {
|
||||
return false, nil
|
||||
return false, fmt.Errorf("does not move volume in remove storage")
|
||||
}
|
||||
|
||||
if candidateVolume.ReplicaPlacement > 0 {
|
||||
|
||||
@@ -58,7 +58,7 @@ func (c *commandVolumeFixReplication) Do(args []string, commandEnv *CommandEnv,
|
||||
c.collectionPattern = volFixReplicationCommand.String("collectionPattern", "", "match with wildcard characters '*' and '?'")
|
||||
skipChange := volFixReplicationCommand.Bool("n", false, "skip the changes")
|
||||
noDelete := volFixReplicationCommand.Bool("noDelete", false, "Do not delete over-replicated volumes, only fix under-replication")
|
||||
retryCount := volFixReplicationCommand.Int("retry", 0, "how many times to retry")
|
||||
retryCount := volFixReplicationCommand.Int("retry", 5, "how many times to retry")
|
||||
volumesPerStep := volFixReplicationCommand.Int("volumesPerStep", 0, "how many volumes to fix in one cycle")
|
||||
|
||||
if err = volFixReplicationCommand.Parse(args); err != nil {
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
@@ -164,6 +163,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
|
||||
|
||||
if *c.findMissingChunksInFiler {
|
||||
// collect all filer file ids and paths
|
||||
|
||||
if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectCutoffFromAtNs); err != nil {
|
||||
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
|
||||
}
|
||||
@@ -409,7 +409,7 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
|
||||
}
|
||||
buf.Write(resp.FileContent)
|
||||
}
|
||||
if vinfo.isReadOnly == false {
|
||||
if !vinfo.isReadOnly {
|
||||
index, err := idx.FirstInvalidIndex(buf.Bytes(),
|
||||
func(key types.NeedleId, offset types.Offset, size types.Size) (bool, error) {
|
||||
resp, err := volumeServerClient.ReadNeedleMeta(context.Background(), &volume_server_pb.ReadNeedleMetaRequest{
|
||||
@@ -419,12 +419,12 @@ func (c *commandVolumeFsck) collectOneVolumeFileIds(dataNodeId string, volumeId
|
||||
Size: int32(size),
|
||||
})
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("to read needle meta with id %d from volume %d with error %v", key, volumeId, err)
|
||||
return false, fmt.Errorf("read needle meta with id %d from volume %d: %v", key, volumeId, err)
|
||||
}
|
||||
return resp.AppendAtNs <= cutoffFrom, nil
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v", volumeId, err)
|
||||
fmt.Fprintf(c.writer, "Failed to search for last valid index on volume %d with error %v\n", volumeId, err)
|
||||
} else {
|
||||
buf.Truncate(index * types.NeedleMapEntrySize)
|
||||
}
|
||||
@@ -535,7 +535,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
_, err = ioutil.ReadAll(resp.Body)
|
||||
_, err = io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
fmt.Fprintf(c.writer, "DELETE response error: %v\n", err)
|
||||
}
|
||||
|
||||
Loaded 100 of 113 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user