mirror of
https://github.com/seaweedfs/seaweedfs.git
synced 2026-11-22 12:17:04 +01:00
Compare commits
94
Commits
message_send
...
3.32
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
758d70bdc0 | ||
|
|
de286fe662 | ||
|
|
52e0a88a15 | ||
|
|
949607fe90 | ||
|
|
e20f0dbd2d | ||
|
|
ee38ab8581 | ||
|
|
7836f7574e | ||
|
|
f476cf3403 | ||
|
|
5c2eb602c9 | ||
|
|
2f72103c83 | ||
|
|
f19c9e3d9d | ||
|
|
9c8678ded9 | ||
|
|
e05637c42c | ||
|
|
dcd0743a35 | ||
|
|
a5b867af69 | ||
|
|
1f7e52c63e | ||
|
|
f5d4952d73 | ||
|
|
f95c25e113 | ||
|
|
9a339a9cfb | ||
|
|
cacc3e883b | ||
|
|
bf5e45b66a | ||
|
|
401315f337 | ||
|
|
a05725aea6 | ||
|
|
e55076c46f | ||
|
|
cea73ac008 | ||
|
|
dff85e9c71 | ||
|
|
8aec786a6d | ||
|
|
0d817bc347 | ||
|
|
84c401e693 | ||
|
|
d21e2f523d | ||
|
|
c34f04b35b | ||
|
|
f6c8720ded | ||
|
|
50ed9456ff | ||
|
|
0aa947913a | ||
|
|
358d9fb109 | ||
|
|
3fbf68de12 | ||
|
|
c4fa148ee6 | ||
|
|
6bbc1d4026 | ||
|
|
43175e7fe8 | ||
|
|
7617567c02 | ||
|
|
5b28c3f728 | ||
|
|
3550692afc | ||
|
|
0711870f43 | ||
|
|
109dc7fdf6 | ||
|
|
fc54afdba9 | ||
|
|
0623bf582e | ||
|
|
98dc1e5c15 | ||
|
|
049f040c3c | ||
|
|
124b97f6d7 | ||
|
|
1e27d36576 | ||
|
|
f9383aa726 | ||
|
|
b7de4a967e | ||
|
|
b9d8a837af | ||
|
|
5db25a8f2a | ||
|
|
f8d3ff466d | ||
|
|
a522507f95 | ||
|
|
b9933d5589 | ||
|
|
36daa7709d | ||
|
|
0f522da367 | ||
|
|
ea271600ec | ||
|
|
0452ae6a6c | ||
|
|
ec46a34f33 | ||
|
|
18d92dbd45 | ||
|
|
93d15488e4 | ||
|
|
595db62fe7 | ||
|
|
8ef802c259 | ||
|
|
5b106630aa | ||
|
|
1cdbc8beb5 | ||
|
|
d3f89af079 | ||
|
|
707abd5b2d | ||
|
|
29632d5a34 | ||
|
|
e9584d9661 | ||
|
|
6fa3d0cc46 | ||
|
|
c3487d3da7 | ||
|
|
ba0db28dbb | ||
|
|
8e81619d02 | ||
|
|
620be2be16 | ||
|
|
56c94cc08e | ||
|
|
3de1e19780 | ||
|
|
5e9039d728 | ||
|
|
e00a12b099 | ||
|
|
aacdcc4cad | ||
|
|
301b678147 | ||
|
|
47fd5d3fa1 | ||
|
|
7ae51d1ec6 | ||
|
|
190afcc3fd | ||
|
|
faa6167b6b | ||
|
|
b6d7556dda | ||
|
|
ea15476b06 | ||
|
|
53c7161f27 | ||
|
|
30200cb6af | ||
|
|
b5d99110d1 | ||
|
|
6b93148f9c | ||
|
|
d1357c1a3a |
No files matched your search
@@ -38,7 +38,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
|
||||
- name: Set BUILD_TIME env
|
||||
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
|
||||
@@ -28,7 +28,7 @@ jobs:
|
||||
# Steps represent a sequence of tasks that will be executed as part of the job
|
||||
steps:
|
||||
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
|
||||
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
- name: Go Release Binaries Normal Volume Size
|
||||
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
|
||||
with:
|
||||
|
||||
@@ -18,7 +18,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -17,7 +17,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
steps:
|
||||
-
|
||||
name: Checkout
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
-
|
||||
name: Docker meta
|
||||
id: docker_meta
|
||||
|
||||
@@ -9,6 +9,6 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: 'Checkout Repository'
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
|
||||
- name: 'Dependency Review'
|
||||
uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
|
||||
uses: actions/dependency-review-action@375c5370086bfff256c37f8beec0f437e2e72ae1
|
||||
@@ -30,7 +30,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
|
||||
@@ -27,7 +27,7 @@ jobs:
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
|
||||
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
+17
-17
@@ -39,63 +39,63 @@ s3tests_build:
|
||||
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
|
||||
|
||||
dev: build
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_race: binary_race
|
||||
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_tls: build certstrap
|
||||
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up
|
||||
|
||||
dev_mount: build
|
||||
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-mount-compose.yml -p seaweedfs up
|
||||
|
||||
run_image: build
|
||||
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
|
||||
|
||||
profile_mount: build
|
||||
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
|
||||
|
||||
k8s: build
|
||||
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-k8s-compose.yml -p seaweedfs up
|
||||
|
||||
dev_registry: build
|
||||
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
|
||||
|
||||
dev_replicate: build
|
||||
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
|
||||
|
||||
dev_auditlog: build
|
||||
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up
|
||||
|
||||
dev_nextcloud: build
|
||||
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
|
||||
|
||||
cluster: build
|
||||
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-cluster-compose.yml -p seaweedfs up
|
||||
|
||||
2clusters: build
|
||||
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-clusters-compose.yml -p seaweedfs up
|
||||
|
||||
2mount: build
|
||||
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
|
||||
|
||||
hashicorp_raft: build
|
||||
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
|
||||
|
||||
s3tests: build s3tests_build
|
||||
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
|
||||
docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
|
||||
|
||||
filer_etcd: build
|
||||
docker stack deploy -c compose/swarm-etcd.yml fs
|
||||
|
||||
test_etcd: build
|
||||
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
|
||||
docker compose -f compose/test-etcd-filer.yml -p seaweedfs up
|
||||
|
||||
test_ydb: tags = ydb
|
||||
test_ydb: build
|
||||
export
|
||||
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
|
||||
docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
|
||||
|
||||
clean:
|
||||
rm ./weed
|
||||
|
||||
+12
-2
@@ -1,5 +1,15 @@
|
||||
# Docker
|
||||
|
||||
## Compose V2
|
||||
SeaweedFS now uses the `v2` syntax `docker compose`
|
||||
|
||||
If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading.
|
||||
|
||||
Confirm your system has docker compose v2 with a version check
|
||||
```bash
|
||||
$ docker compose version
|
||||
Docker Compose version v2.10.2
|
||||
```
|
||||
|
||||
## Try it out
|
||||
|
||||
@@ -7,7 +17,7 @@
|
||||
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
docker compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
```
|
||||
|
||||
@@ -17,7 +27,7 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
|
||||
|
||||
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
|
||||
|
||||
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up
|
||||
|
||||
```
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
s3:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master0:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
server1:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master0:
|
||||
@@ -6,7 +6,7 @@ services:
|
||||
ports:
|
||||
- 9333:9333
|
||||
- 19333:19333
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/0:/data
|
||||
environment:
|
||||
@@ -18,7 +18,7 @@ services:
|
||||
ports:
|
||||
- 9334:9334
|
||||
- 19334:19334
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/1:/data
|
||||
environment:
|
||||
@@ -30,7 +30,7 @@ services:
|
||||
ports:
|
||||
- 9335:9335
|
||||
- 19335:19335
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
|
||||
volumes:
|
||||
- ./master/2:/data
|
||||
environment:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
etcd:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
ydb:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
version: '2'
|
||||
version: '3.9'
|
||||
|
||||
services:
|
||||
master:
|
||||
|
||||
@@ -5,16 +5,16 @@ go 1.19
|
||||
require (
|
||||
cloud.google.com/go v0.104.0 // indirect
|
||||
cloud.google.com/go/pubsub v1.25.1
|
||||
cloud.google.com/go/storage v1.26.0
|
||||
cloud.google.com/go/storage v1.27.0
|
||||
github.com/Azure/azure-pipeline-go v0.2.3
|
||||
github.com/Azure/azure-storage-blob-go v0.15.0
|
||||
github.com/Shopify/sarama v1.36.0
|
||||
github.com/aws/aws-sdk-go v1.44.100
|
||||
github.com/Shopify/sarama v1.37.2
|
||||
github.com/aws/aws-sdk-go v1.44.114
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/bwmarrin/snowflake v0.3.0
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
|
||||
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
@@ -27,7 +27,7 @@ require (
|
||||
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
|
||||
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
|
||||
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
|
||||
github.com/fclairamb/ftpserverlib v0.19.1
|
||||
github.com/fclairamb/ftpserverlib v0.20.0
|
||||
github.com/fsnotify/fsnotify v1.5.4 // indirect
|
||||
github.com/go-errors/errors v1.1.1 // indirect
|
||||
github.com/go-redis/redis/v8 v8.11.5
|
||||
@@ -43,10 +43,10 @@ require (
|
||||
github.com/google/go-cmp v0.5.8 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/google/wire v0.5.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.5.1 // indirect
|
||||
github.com/gorilla/mux v1.8.0
|
||||
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
|
||||
github.com/hashicorp/errwrap v1.0.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-uuid v1.0.3 // indirect
|
||||
github.com/hashicorp/hcl v1.0.0 // indirect
|
||||
@@ -57,8 +57,8 @@ require (
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/karlseguin/ccache/v2 v2.0.8
|
||||
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
|
||||
github.com/klauspost/compress v1.15.9 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.0
|
||||
github.com/klauspost/compress v1.15.11 // indirect
|
||||
github.com/klauspost/reedsolomon v1.11.1
|
||||
github.com/kurin/blazer v0.5.3
|
||||
github.com/lib/pq v1.10.7
|
||||
github.com/linxGnu/grocksdb v1.7.7
|
||||
@@ -66,7 +66,7 @@ require (
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-ieproxy v0.0.9 // indirect
|
||||
github.com/mattn/go-isatty v0.0.16 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
@@ -85,7 +85,7 @@ require (
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
|
||||
github.com/seaweedfs/goexif v2.0.0+incompatible
|
||||
github.com/seaweedfs/raft v1.1.0
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
github.com/sirupsen/logrus v1.9.0 // indirect
|
||||
github.com/spf13/afero v1.9.2 // indirect
|
||||
github.com/spf13/cast v1.5.0 // indirect
|
||||
github.com/spf13/jwalterweatherman v1.1.0 // indirect
|
||||
@@ -107,24 +107,24 @@ require (
|
||||
github.com/xdg-go/stringprep v1.0.3 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
|
||||
go.etcd.io/etcd/client/v3 v3.5.5
|
||||
go.mongodb.org/mongo-driver v1.10.2
|
||||
go.mongodb.org/mongo-driver v1.10.3
|
||||
go.opencensus.io v0.23.0 // indirect
|
||||
gocloud.dev v0.26.0
|
||||
gocloud.dev/pubsub/natspubsub v0.26.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.26.0
|
||||
gocloud.dev v0.27.0
|
||||
gocloud.dev/pubsub/natspubsub v0.27.0
|
||||
gocloud.dev/pubsub/rabbitpubsub v0.27.0
|
||||
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
|
||||
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
|
||||
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
|
||||
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
|
||||
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
|
||||
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
|
||||
golang.org/x/net v0.0.0-20220927171203-f486391704dc
|
||||
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
|
||||
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/tools v0.1.12
|
||||
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
|
||||
google.golang.org/api v0.96.0
|
||||
google.golang.org/api v0.98.0
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
|
||||
google.golang.org/grpc v1.49.0
|
||||
google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
|
||||
google.golang.org/grpc v1.50.0
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
modernc.org/b v1.0.0 // indirect
|
||||
@@ -141,17 +141,18 @@ require (
|
||||
|
||||
require (
|
||||
github.com/Jille/raft-grpc-transport v1.2.0
|
||||
github.com/arangodb/go-driver v1.3.3
|
||||
github.com/arangodb/go-driver v1.4.0
|
||||
github.com/armon/go-metrics v0.4.1
|
||||
github.com/fluent/fluent-logger-golang v1.9.0
|
||||
github.com/google/flatbuffers v2.0.8+incompatible
|
||||
github.com/google/flatbuffers v22.9.29+incompatible
|
||||
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
|
||||
github.com/hashicorp/raft v1.3.10
|
||||
github.com/hashicorp/raft v1.3.11
|
||||
github.com/hashicorp/raft-boltdb/v2 v2.2.2
|
||||
github.com/rabbitmq/amqp091-go v1.5.0
|
||||
github.com/tikv/client-go/v2 v2.0.1
|
||||
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
|
||||
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.3
|
||||
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
|
||||
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
|
||||
)
|
||||
|
||||
@@ -159,31 +160,30 @@ require (
|
||||
cloud.google.com/go/compute v1.7.0 // indirect
|
||||
cloud.google.com/go/iam v0.3.0 // indirect
|
||||
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
|
||||
github.com/armon/go-metrics v0.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
|
||||
github.com/aws/smithy-go v1.11.2 // indirect
|
||||
github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
|
||||
github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
|
||||
github.com/aws/smithy-go v1.12.0 // indirect
|
||||
github.com/benbjohnson/clock v1.1.0 // indirect
|
||||
github.com/boltdb/bolt v1.3.1 // indirect
|
||||
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
|
||||
github.com/d4l3k/messagediff v1.2.1 // indirect
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
|
||||
github.com/fatih/color v1.13.0 // indirect
|
||||
github.com/fclairamb/go-log v0.3.0 // indirect
|
||||
github.com/fclairamb/go-log v0.4.1 // indirect
|
||||
github.com/gogo/protobuf v1.3.2 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
|
||||
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
|
||||
github.com/hashicorp/go-hclog v1.2.0 // indirect
|
||||
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
|
||||
github.com/hashicorp/go-msgpack v1.1.5 // indirect
|
||||
@@ -194,17 +194,17 @@ require (
|
||||
github.com/jonboulle/clockwork v0.2.2 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.7 // indirect
|
||||
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
|
||||
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
|
||||
github.com/montanaflynn/stats v0.6.6 // indirect
|
||||
github.com/nats-io/nats.go v1.16.0 // indirect
|
||||
github.com/nats-io/nkeys v0.3.0 // indirect
|
||||
github.com/nats-io/nuid v1.0.1 // indirect
|
||||
github.com/opentracing/opentracing-go v1.2.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
|
||||
github.com/philhofer/fwd v1.1.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.15 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.17 // indirect
|
||||
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
|
||||
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
|
||||
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
|
||||
@@ -216,7 +216,7 @@ require (
|
||||
github.com/tinylib/msgp v1.1.6 // indirect
|
||||
github.com/twmb/murmur3 v1.1.3 // indirect
|
||||
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6 // indirect
|
||||
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
|
||||
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
|
||||
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
|
||||
go.etcd.io/bbolt v1.3.6 // indirect
|
||||
@@ -224,7 +224,7 @@ require (
|
||||
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
|
||||
go.uber.org/atomic v1.9.0 // indirect
|
||||
go.uber.org/multierr v1.8.0 // indirect
|
||||
go.uber.org/zap v1.21.0 // indirect
|
||||
go.uber.org/zap v1.22.0 // indirect
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
apiVersion: v1
|
||||
description: SeaweedFS
|
||||
name: seaweedfs
|
||||
appVersion: "3.29"
|
||||
version: "3.29"
|
||||
appVersion: "3.32"
|
||||
version: "3.32"
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -14,7 +14,7 @@
|
||||
</parent>
|
||||
|
||||
<properties>
|
||||
<protobuf.version>3.16.1</protobuf.version>
|
||||
<protobuf.version>3.16.3</protobuf.version>
|
||||
<!-- follow https://github.com/grpc/grpc-java -->
|
||||
<grpc.version>1.23.0</grpc.version>
|
||||
<guava.version>30.0-jre</guava.version>
|
||||
@@ -159,7 +159,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -154,7 +154,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
|
||||
<parent>
|
||||
<groupId>org.sonatype.oss</groupId>
|
||||
@@ -128,7 +128,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -119,9 +119,8 @@ public class SeaweedInputStream extends InputStream {
|
||||
|
||||
long bytesRead = 0;
|
||||
int len = buf.remaining();
|
||||
int start = (int) this.position;
|
||||
if (start + len <= entry.getContent().size()) {
|
||||
entry.getContent().substring(start, start + len).copyTo(buf);
|
||||
if (this.position< Integer.MAX_VALUE && (this.position + len )<= entry.getContent().size()) {
|
||||
entry.getContent().substring((int)this.position, (int)(this.position + len)).copyTo(buf);
|
||||
} else {
|
||||
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
|
||||
}
|
||||
|
||||
@@ -14,7 +14,9 @@ import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.MessageDigest;
|
||||
import java.util.List;
|
||||
import java.util.Base64;
|
||||
|
||||
public class SeaweedWrite {
|
||||
|
||||
@@ -123,13 +125,20 @@ public class SeaweedWrite {
|
||||
final byte[] bytes,
|
||||
final long bytesOffset, final long bytesLength,
|
||||
byte[] cipherKey) throws IOException {
|
||||
MessageDigest md = null;
|
||||
try {
|
||||
md = MessageDigest.getInstance("MD5");
|
||||
} catch (java.security.NoSuchAlgorithmException e) {
|
||||
}
|
||||
|
||||
InputStream inputStream = null;
|
||||
if (cipherKey == null || cipherKey.length == 0) {
|
||||
md.update(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
|
||||
} else {
|
||||
try {
|
||||
byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
|
||||
md.update(encryptedBytes);
|
||||
inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
|
||||
} catch (Exception e) {
|
||||
throw new IOException("fail to encrypt data", e);
|
||||
@@ -140,6 +149,7 @@ public class SeaweedWrite {
|
||||
if (auth != null && auth.length() != 0) {
|
||||
post.addHeader("Authorization", "BEARER " + auth);
|
||||
}
|
||||
post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
|
||||
|
||||
post.setEntity(MultipartEntityBuilder.create()
|
||||
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
|
||||
@@ -149,6 +159,13 @@ public class SeaweedWrite {
|
||||
CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
|
||||
|
||||
try {
|
||||
if (response.getStatusLine().getStatusCode() / 100 != 2) {
|
||||
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
|
||||
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
|
||||
} else {
|
||||
throw new IOException(response.getStatusLine().getReasonPhrase());
|
||||
}
|
||||
}
|
||||
|
||||
String etag = response.getLastHeader("ETag").getValue();
|
||||
|
||||
|
||||
@@ -11,13 +11,13 @@
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.github.chrislusf</groupId>
|
||||
<artifactId>seaweedfs-hadoop2-client</artifactId>
|
||||
<version>3.13</version>
|
||||
<version>3.30</version>
|
||||
<scope>compile</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -132,11 +132,11 @@
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-mapreduce-client-app</artifactId>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-yarn-api</artifactId>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>2.10.1</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -167,10 +167,6 @@
|
||||
<artifactId>commons-math3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>xmlenc</artifactId>
|
||||
<groupId>xmlenc</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-io</artifactId>
|
||||
<groupId>commons-io</groupId>
|
||||
@@ -184,20 +180,28 @@
|
||||
<groupId>commons-collections</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>servlet-api</artifactId>
|
||||
<artifactId>javax.servlet-api</artifactId>
|
||||
<groupId>javax.servlet</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>javax.activation-api</artifactId>
|
||||
<groupId>javax.activation</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-server</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-util</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-sslengine</artifactId>
|
||||
<groupId>org.mortbay.jetty</groupId>
|
||||
<artifactId>jetty-servlet</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jetty-webapp</artifactId>
|
||||
<groupId>org.eclipse.jetty</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jsp-api</artifactId>
|
||||
@@ -207,6 +211,10 @@
|
||||
<artifactId>jersey-core</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-servlet</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jersey-json</artifactId>
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
@@ -216,49 +224,37 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jets3t</artifactId>
|
||||
<groupId>net.java.dev.jets3t</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang</artifactId>
|
||||
<groupId>commons-lang</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration</artifactId>
|
||||
<groupId>commons-configuration</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-digester</artifactId>
|
||||
<groupId>commons-digester</groupId>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
<groupId>commons-beanutils</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-configuration2</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-lang3</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<artifactId>commons-text</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-core-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-mapper-asl</artifactId>
|
||||
<groupId>org.codehaus.jackson</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>avro</artifactId>
|
||||
<groupId>org.apache.avro</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>re2j</artifactId>
|
||||
<groupId>com.google.re2j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-auth</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -287,6 +283,14 @@
|
||||
<artifactId>commons-compress</artifactId>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>kerb-simplekdc</artifactId>
|
||||
<groupId>org.apache.kerby</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>stax2-api</artifactId>
|
||||
<groupId>org.codehaus.woodstox</groupId>
|
||||
@@ -295,6 +299,10 @@
|
||||
<artifactId>woodstox-core</artifactId>
|
||||
<groupId>com.fasterxml.woodstox</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>dnsjava</artifactId>
|
||||
<groupId>dnsjava</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>hadoop-annotations</artifactId>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
@@ -309,7 +317,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>2.10.1</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -108,7 +108,7 @@
|
||||
</plugin>
|
||||
<plugin>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
@@ -124,7 +124,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-client</artifactId>
|
||||
<version>3.2.3</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -156,7 +156,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-common</artifactId>
|
||||
<version>3.2.3</version>
|
||||
<version>3.2.4</version>
|
||||
<scope>provided</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
@@ -224,8 +224,8 @@
|
||||
<groupId>com.sun.jersey</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>log4j</artifactId>
|
||||
<groupId>log4j</groupId>
|
||||
<artifactId>reload4j</artifactId>
|
||||
<groupId>ch.qos.reload4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>commons-beanutils</artifactId>
|
||||
@@ -244,7 +244,7 @@
|
||||
<groupId>org.apache.commons</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<artifactId>slf4j-log4j12</artifactId>
|
||||
<artifactId>slf4j-reload4j</artifactId>
|
||||
<groupId>org.slf4j</groupId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
@@ -317,7 +317,7 @@
|
||||
</snapshotRepository>
|
||||
</distributionManagement>
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.3</hadoop.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
</project>
|
||||
@@ -5,7 +5,7 @@
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<properties>
|
||||
<seaweedfs.client.version>3.13</seaweedfs.client.version>
|
||||
<seaweedfs.client.version>3.30</seaweedfs.client.version>
|
||||
<hadoop.version>3.2.4</hadoop.version>
|
||||
</properties>
|
||||
|
||||
@@ -129,7 +129,7 @@
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-javadoc-plugin</artifactId>
|
||||
<version>2.9.1</version>
|
||||
<version>3.0.1</version>
|
||||
<executions>
|
||||
<execution>
|
||||
<id>attach-javadocs</id>
|
||||
|
||||
@@ -37,11 +37,16 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
||||
|
||||
var lastLogTsNs = time.Now().UnixNano()
|
||||
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
if *option.storageClass == "" {
|
||||
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
|
||||
} else {
|
||||
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
|
||||
if resp.EventNotification.NewEntry != nil {
|
||||
if *option.storageClass == "" {
|
||||
if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
|
||||
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
|
||||
}
|
||||
} else {
|
||||
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
|
||||
}
|
||||
}
|
||||
|
||||
processor.AddSyncJob(resp)
|
||||
return nil
|
||||
}, 3*time.Second, func(counter int64, lastTsNs int64) error {
|
||||
|
||||
+10
-1
@@ -47,7 +47,10 @@ func runFuse(cmd *Command, args []string) bool {
|
||||
for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
|
||||
option.WriteByte(rawArgs[i])
|
||||
}
|
||||
options = append(options, parameter{option.String(), "true"})
|
||||
// ignore "-o"
|
||||
if option.String() != "o" {
|
||||
options = append(options, parameter{option.String(), "true"})
|
||||
}
|
||||
option.Reset()
|
||||
|
||||
// equal separator start option with pending value
|
||||
@@ -190,6 +193,12 @@ func runFuse(cmd *Command, args []string) bool {
|
||||
}
|
||||
case "fusermount.path":
|
||||
fusermountPath = parameter.value
|
||||
default:
|
||||
t := parameter.name
|
||||
if parameter.value != "true" {
|
||||
t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
|
||||
}
|
||||
mountOptions.extraOptions = append(mountOptions.extraOptions, t)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package command
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
hashicorpRaft "github.com/hashicorp/raft"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
@@ -253,6 +254,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
|
||||
go httpS.Serve(masterListener)
|
||||
}
|
||||
|
||||
grace.OnInterrupt(ms.Shutdown)
|
||||
grace.OnInterrupt(grpcS.Stop)
|
||||
grace.OnReload(func() {
|
||||
if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
|
||||
ms.Topo.HashicorpRaft.LeadershipTransfer()
|
||||
}
|
||||
})
|
||||
select {}
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ type MountOptions struct {
|
||||
debugPort *int
|
||||
localSocket *string
|
||||
disableXAttr *bool
|
||||
extraOptions []string
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
|
||||
// mount fuse
|
||||
fuseMountOptions := &fuse.MountOptions{
|
||||
AllowOther: *option.allowOthers,
|
||||
Options: nil,
|
||||
Options: option.extraOptions,
|
||||
MaxBackground: 128,
|
||||
MaxWrite: 1024 * 1024 * 2,
|
||||
MaxReadAhead: 1024 * 1024 * 2,
|
||||
|
||||
@@ -92,10 +92,11 @@ func init() {
|
||||
masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
|
||||
masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
|
||||
masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
|
||||
masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
|
||||
masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
|
||||
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
|
||||
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
|
||||
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
|
||||
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
|
||||
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
|
||||
|
||||
@@ -131,7 +132,7 @@ func init() {
|
||||
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
|
||||
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
|
||||
serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
|
||||
|
||||
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
|
||||
|
||||
@@ -98,7 +98,7 @@ func init() {
|
||||
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
|
||||
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
|
||||
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
|
||||
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
|
||||
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
|
||||
}
|
||||
|
||||
@@ -265,6 +265,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
|
||||
// starting the cluster http server
|
||||
clusterHttpServer := v.startClusterHttpService(volumeMux)
|
||||
|
||||
grace.OnReload(volumeServer.LoadNewVolumes)
|
||||
|
||||
stopChan := make(chan bool)
|
||||
grace.OnInterrupt(func() {
|
||||
fmt.Println("volume server has been killed")
|
||||
|
||||
@@ -35,6 +35,22 @@ func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error
|
||||
|
||||
func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
if err = store.doInsertEntry(ctx, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
if name != "" {
|
||||
_, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedisStore) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
@@ -49,21 +65,12 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
}
|
||||
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
if name != "" {
|
||||
_, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result()
|
||||
if err != nil {
|
||||
return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
return store.InsertEntry(ctx, entry)
|
||||
return store.doInsertEntry(ctx, entry)
|
||||
}
|
||||
|
||||
func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
|
||||
@@ -47,17 +47,8 @@ func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) erro
|
||||
|
||||
func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
|
||||
value = util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
if err = store.doInsertEntry(ctx, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
@@ -74,9 +65,25 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
|
||||
value = util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
return store.InsertEntry(ctx, entry)
|
||||
return store.doInsertEntry(ctx, entry)
|
||||
}
|
||||
|
||||
func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
|
||||
@@ -3,11 +3,11 @@ package redis3
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/go-redsync/redsync/v4"
|
||||
"time"
|
||||
|
||||
"github.com/go-redis/redis/v8"
|
||||
|
||||
redsync "github.com/go-redsync/redsync/v4"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
@@ -35,17 +35,8 @@ func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) erro
|
||||
|
||||
func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
|
||||
value = util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
if err = store.doInsertEntry(ctx, entry); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dir, name := entry.FullPath.DirAndName()
|
||||
@@ -59,9 +50,25 @@ func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedis3Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
|
||||
value, err := entry.EncodeAttributesAndChunks()
|
||||
if err != nil {
|
||||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
|
||||
}
|
||||
|
||||
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
|
||||
value = util.MaybeGzipData(value)
|
||||
}
|
||||
|
||||
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
|
||||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
|
||||
|
||||
return store.InsertEntry(ctx, entry)
|
||||
return store.doInsertEntry(ctx, entry)
|
||||
}
|
||||
|
||||
func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
|
||||
|
||||
@@ -10,7 +10,11 @@ import (
|
||||
)
|
||||
|
||||
func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
|
||||
if err := jsonpb.Unmarshal(content, config); err != nil {
|
||||
options := &jsonpb.UnmarshalOptions{
|
||||
DiscardUnknown: true,
|
||||
AllowPartial: true,
|
||||
}
|
||||
if err := options.Unmarshal(content, config); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -63,15 +63,19 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
|
||||
var dbErr error
|
||||
store.DB, dbErr = sql.Open("sqlite", dbFile)
|
||||
if dbErr != nil {
|
||||
store.DB.Close()
|
||||
store.DB = nil
|
||||
return fmt.Errorf("can not connect to %s error:%v", dbFile, err)
|
||||
if store.DB != nil {
|
||||
store.DB.Close()
|
||||
store.DB = nil
|
||||
}
|
||||
return fmt.Errorf("can not connect to %s error:%v", dbFile, dbErr)
|
||||
}
|
||||
|
||||
if err = store.DB.Ping(); err != nil {
|
||||
return fmt.Errorf("connect to %s error:%v", dbFile, err)
|
||||
}
|
||||
|
||||
store.DB.SetMaxOpenConns(1)
|
||||
|
||||
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
|
||||
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
|
||||
}
|
||||
|
||||
@@ -9,37 +9,41 @@ import (
|
||||
)
|
||||
|
||||
func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *Entry)) (err error) {
|
||||
|
||||
K := 5
|
||||
|
||||
var jobQueueWg sync.WaitGroup
|
||||
queue := util.NewQueue()
|
||||
jobQueueWg.Add(1)
|
||||
queue.Enqueue(parentPath)
|
||||
var isTerminating bool
|
||||
terminates := make([]chan bool, K)
|
||||
|
||||
for i := 0; i < K; i++ {
|
||||
go func() {
|
||||
terminates[i] = make(chan bool)
|
||||
go func(j int) {
|
||||
for {
|
||||
if isTerminating {
|
||||
break
|
||||
select {
|
||||
case <-terminates[j]:
|
||||
return
|
||||
default:
|
||||
t := queue.Dequeue()
|
||||
if t == nil {
|
||||
time.Sleep(329 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
dir := t.(util.FullPath)
|
||||
processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn)
|
||||
if processErr != nil {
|
||||
err = processErr
|
||||
}
|
||||
jobQueueWg.Done()
|
||||
}
|
||||
t := queue.Dequeue()
|
||||
if t == nil {
|
||||
time.Sleep(329 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
dir := t.(util.FullPath)
|
||||
processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn)
|
||||
if processErr != nil {
|
||||
err = processErr
|
||||
}
|
||||
jobQueueWg.Done()
|
||||
}
|
||||
}()
|
||||
}(i)
|
||||
}
|
||||
jobQueueWg.Wait()
|
||||
isTerminating = true
|
||||
for i := 0; i < K; i++ {
|
||||
close(terminates[i])
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ message Identity {
|
||||
string name = 1;
|
||||
repeated Credential credentials = 2;
|
||||
repeated string actions = 3;
|
||||
string accountId = 4;
|
||||
}
|
||||
|
||||
message Credential {
|
||||
|
||||
+30
-20
@@ -75,6 +75,7 @@ type Identity struct {
|
||||
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
|
||||
Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
|
||||
Actions []string `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
|
||||
AccountId string `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"`
|
||||
}
|
||||
|
||||
func (x *Identity) Reset() {
|
||||
@@ -130,6 +131,13 @@ func (x *Identity) GetActions() []string {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (x *Identity) GetAccountId() string {
|
||||
if x != nil {
|
||||
return x.AccountId
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type Credential struct {
|
||||
state protoimpl.MessageState
|
||||
sizeCache protoimpl.SizeCache
|
||||
@@ -193,26 +201,28 @@ var file_iam_proto_rawDesc = []byte{
|
||||
0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e,
|
||||
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52,
|
||||
0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x6e, 0x0a, 0x08, 0x49,
|
||||
0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
|
||||
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63,
|
||||
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
|
||||
0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e,
|
||||
0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
|
||||
0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03,
|
||||
0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4a, 0x0a, 0x0a, 0x43,
|
||||
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63,
|
||||
0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61,
|
||||
0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72,
|
||||
0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65,
|
||||
0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65,
|
||||
0x65, 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73,
|
||||
0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65,
|
||||
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08,
|
||||
0x49, 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
|
||||
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73,
|
||||
0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
|
||||
0x2f, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08,
|
||||
0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
|
||||
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b,
|
||||
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
|
||||
0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65,
|
||||
0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
|
||||
0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20,
|
||||
0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09,
|
||||
0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
|
||||
0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72,
|
||||
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65,
|
||||
0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63,
|
||||
0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65,
|
||||
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63,
|
||||
0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
|
||||
0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d,
|
||||
0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61,
|
||||
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49,
|
||||
0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
|
||||
0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65,
|
||||
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
|
||||
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -400,7 +400,8 @@ message RaftListClusterServersResponse {
|
||||
message ClusterServers {
|
||||
string id = 1;
|
||||
string address = 2;
|
||||
string suffrage = 3; //
|
||||
string suffrage = 3;
|
||||
bool isLeader = 4;
|
||||
}
|
||||
repeated ClusterServers cluster_servers = 1;
|
||||
}
|
||||
+120
-110
@@ -3817,7 +3817,8 @@ type RaftListClusterServersResponse_ClusterServers struct {
|
||||
|
||||
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
|
||||
Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
|
||||
Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"` //
|
||||
Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"`
|
||||
IsLeader bool `protobuf:"varint,4,opt,name=isLeader,proto3" json:"isLeader,omitempty"`
|
||||
}
|
||||
|
||||
func (x *RaftListClusterServersResponse_ClusterServers) Reset() {
|
||||
@@ -3873,6 +3874,13 @@ func (x *RaftListClusterServersResponse_ClusterServers) GetSuffrage() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x *RaftListClusterServersResponse_ClusterServers) GetIsLeader() bool {
|
||||
if x != nil {
|
||||
return x.IsLeader
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
var File_master_proto protoreflect.FileDescriptor
|
||||
|
||||
var file_master_proto_rawDesc = []byte{
|
||||
@@ -4443,7 +4451,7 @@ var file_master_proto_rawDesc = []byte{
|
||||
0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xdb, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf7, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
|
||||
0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65,
|
||||
0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0f, 0x63, 0x6c,
|
||||
0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20,
|
||||
@@ -4451,122 +4459,124 @@ var file_master_proto_rawDesc = []byte{
|
||||
0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53,
|
||||
0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43,
|
||||
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x63,
|
||||
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x56, 0x0a,
|
||||
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x72, 0x0a,
|
||||
0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12,
|
||||
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12,
|
||||
0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
|
||||
0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x75, 0x66,
|
||||
0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x75, 0x66,
|
||||
0x66, 0x72, 0x61, 0x67, 0x65, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
|
||||
0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65,
|
||||
0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48,
|
||||
0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d,
|
||||
0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e,
|
||||
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
|
||||
0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43,
|
||||
0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
|
||||
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73,
|
||||
0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e,
|
||||
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74,
|
||||
0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
|
||||
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61,
|
||||
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
|
||||
0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
|
||||
0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65,
|
||||
0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
|
||||
0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
|
||||
0x72, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a,
|
||||
0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74,
|
||||
0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70,
|
||||
0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
|
||||
0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
|
||||
0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01,
|
||||
0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
|
||||
0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
|
||||
0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
|
||||
0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12,
|
||||
0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69,
|
||||
0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73,
|
||||
0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53,
|
||||
0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
|
||||
0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10,
|
||||
0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
|
||||
0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c,
|
||||
0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74,
|
||||
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44,
|
||||
0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
|
||||
0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e,
|
||||
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
|
||||
0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61,
|
||||
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e,
|
||||
0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
|
||||
0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f,
|
||||
0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56,
|
||||
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75,
|
||||
0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
|
||||
0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71,
|
||||
0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f,
|
||||
0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a,
|
||||
0x16, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
|
||||
0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e,
|
||||
0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65,
|
||||
0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61,
|
||||
0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d,
|
||||
0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64,
|
||||
0x65, 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
|
||||
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56,
|
||||
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b,
|
||||
0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56,
|
||||
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d,
|
||||
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45,
|
||||
0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
|
||||
0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
|
||||
0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
|
||||
0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
|
||||
0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
|
||||
0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61,
|
||||
0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24, 0x2e, 0x6d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72,
|
||||
0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
|
||||
0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c,
|
||||
0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47, 0x65, 0x74,
|
||||
0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74,
|
||||
0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
|
||||
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e,
|
||||
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
|
||||
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x22,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43,
|
||||
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
|
||||
0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
|
||||
0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f,
|
||||
0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a,
|
||||
0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
|
||||
0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61,
|
||||
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c,
|
||||
0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23,
|
||||
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61,
|
||||
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
|
||||
0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
|
||||
0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50,
|
||||
0x69, 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
|
||||
0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61,
|
||||
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70,
|
||||
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
|
||||
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
|
||||
0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66,
|
||||
0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43,
|
||||
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41,
|
||||
0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
|
||||
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
|
||||
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72,
|
||||
0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a,
|
||||
0x10, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65,
|
||||
0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61,
|
||||
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d,
|
||||
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
|
||||
0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73,
|
||||
0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
|
||||
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65,
|
||||
0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73,
|
||||
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
|
||||
0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
|
||||
0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65,
|
||||
0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73,
|
||||
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12,
|
||||
0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67,
|
||||
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
|
||||
0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
|
||||
0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c,
|
||||
0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x28, 0x2e, 0x6d,
|
||||
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73,
|
||||
0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52,
|
||||
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
|
||||
0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74,
|
||||
0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
|
||||
0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65,
|
||||
0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52,
|
||||
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x52, 0x61, 0x66,
|
||||
0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e,
|
||||
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65,
|
||||
0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
|
||||
0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
|
||||
0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
|
||||
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
|
||||
0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76,
|
||||
0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30,
|
||||
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
|
||||
0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77,
|
||||
0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
|
||||
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
|
||||
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68,
|
||||
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
|
||||
0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f,
|
||||
0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
|
||||
0x6f, 0x74, 0x6f, 0x33,
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -279,7 +279,6 @@ message CopyFileResponse {
|
||||
|
||||
message ReadNeedleBlobRequest {
|
||||
uint32 volume_id = 1;
|
||||
uint64 needle_id = 2;
|
||||
int64 offset = 3; // actual offset
|
||||
int32 size = 4;
|
||||
}
|
||||
@@ -430,6 +429,7 @@ message ReadVolumeFileStatusResponse {
|
||||
uint32 compaction_revision = 7;
|
||||
string collection = 8;
|
||||
string disk_type = 9;
|
||||
VolumeInfo volume_info = 10;
|
||||
}
|
||||
|
||||
message DiskStatus {
|
||||
@@ -466,6 +466,7 @@ message VolumeInfo {
|
||||
repeated RemoteFile files = 1;
|
||||
uint32 version = 2;
|
||||
string replication = 3;
|
||||
uint32 BytesOffset = 4;
|
||||
}
|
||||
|
||||
// tiered storage
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -24,15 +24,16 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
|
||||
var wg sync.WaitGroup
|
||||
for chunkIndex, sourceChunk := range sourceChunks {
|
||||
wg.Add(1)
|
||||
go func(chunk *filer_pb.FileChunk, index int) {
|
||||
index, source := chunkIndex, sourceChunk
|
||||
fs.executor.Execute(func() {
|
||||
defer wg.Done()
|
||||
replicatedChunk, e := fs.replicateOneChunk(chunk, path)
|
||||
replicatedChunk, e := fs.replicateOneChunk(source, path)
|
||||
if e != nil {
|
||||
err = e
|
||||
return
|
||||
}
|
||||
replicatedChunks[index] = replicatedChunk
|
||||
}(sourceChunk, chunkIndex)
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ type FilerSink struct {
|
||||
address string
|
||||
writeChunkByFiler bool
|
||||
isIncremental bool
|
||||
executor *util.LimitedConcurrentExecutor
|
||||
}
|
||||
|
||||
func init() {
|
||||
@@ -83,6 +84,7 @@ func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
|
||||
fs.diskType = diskType
|
||||
fs.grpcDialOption = grpcDialOption
|
||||
fs.writeChunkByFiler = writeChunkByFiler
|
||||
fs.executor = util.NewLimitedConcurrentExecutor(32)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ package s3api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
)
|
||||
|
||||
var IdentityAnonymous *Identity
|
||||
|
||||
type Action string
|
||||
|
||||
type Iam interface {
|
||||
@@ -32,10 +35,15 @@ type IdentityAccessManagement struct {
|
||||
|
||||
type Identity struct {
|
||||
Name string
|
||||
AccountId string
|
||||
Credentials []*Credential
|
||||
Actions []Action
|
||||
}
|
||||
|
||||
func (i *Identity) isAnonymous() bool {
|
||||
return i.Name == s3account.AccountAnonymous.Name
|
||||
}
|
||||
|
||||
type Credential struct {
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
@@ -125,9 +133,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
|
||||
for _, ident := range config.Identities {
|
||||
t := &Identity{
|
||||
Name: ident.Name,
|
||||
AccountId: s3account.AccountAdmin.Id,
|
||||
Credentials: nil,
|
||||
Actions: nil,
|
||||
}
|
||||
|
||||
if ident.Name == s3account.AccountAnonymous.Name {
|
||||
if ident.AccountId != "" && ident.AccountId != s3account.AccountAnonymous.Id {
|
||||
glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
|
||||
}
|
||||
t.AccountId = s3account.AccountAnonymous.Id
|
||||
IdentityAnonymous = t
|
||||
} else {
|
||||
if len(ident.AccountId) > 0 {
|
||||
t.AccountId = ident.AccountId
|
||||
}
|
||||
}
|
||||
|
||||
for _, action := range ident.Actions {
|
||||
t.Actions = append(t.Actions, Action(action))
|
||||
}
|
||||
@@ -139,6 +161,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
|
||||
}
|
||||
identities = append(identities, t)
|
||||
}
|
||||
|
||||
if IdentityAnonymous == nil {
|
||||
IdentityAnonymous = &Identity{
|
||||
Name: s3account.AccountAnonymous.Name,
|
||||
AccountId: s3account.AccountAnonymous.Id,
|
||||
}
|
||||
}
|
||||
iam.m.Lock()
|
||||
// atomically switch
|
||||
iam.identities = identities
|
||||
@@ -173,7 +202,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun
|
||||
iam.m.RLock()
|
||||
defer iam.m.RUnlock()
|
||||
for _, ident := range iam.identities {
|
||||
if ident.Name == "anonymous" {
|
||||
if ident.isAnonymous() {
|
||||
return ident, true
|
||||
}
|
||||
}
|
||||
@@ -259,6 +288,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
|
||||
return identity, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
if !identity.isAnonymous() {
|
||||
r.Header.Set(s3_constants.AmzAccountId, identity.AccountId)
|
||||
}
|
||||
return identity, s3err.ErrNone
|
||||
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
)
|
||||
|
||||
func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) {
|
||||
func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, prefix string, directoriesToWatch []string) {
|
||||
|
||||
processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
|
||||
|
||||
@@ -28,6 +28,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
|
||||
|
||||
_ = s3a.onIamConfigUpdate(dir, fileName, content)
|
||||
_ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content)
|
||||
_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -35,7 +36,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
|
||||
var clientEpoch int32
|
||||
util.RetryForever("followIamChanges", func() error {
|
||||
clientEpoch++
|
||||
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, nil, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
|
||||
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
|
||||
}, func(err error) bool {
|
||||
glog.V(0).Infof("iam follow metadata changes: %v", err)
|
||||
return true
|
||||
@@ -63,3 +64,17 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//reload bucket metadata
|
||||
func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
|
||||
if dir == s3a.option.BucketsPath {
|
||||
if newEntry != nil {
|
||||
s3a.bucketRegistry.LoadBucketMetadata(newEntry)
|
||||
glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry)
|
||||
} else {
|
||||
s3a.bucketRegistry.RemoveBucketMetadata(oldEntry)
|
||||
glog.V(0).Infof("remove bucketMetadata %s/%s", dir, newEntry)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -2,7 +2,9 @@ package s3api
|
||||
|
||||
import (
|
||||
. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
jsonpb "google.golang.org/protobuf/encoding/protojson"
|
||||
@@ -124,5 +126,98 @@ func TestCanDo(t *testing.T) {
|
||||
}
|
||||
assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
|
||||
assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
|
||||
}
|
||||
|
||||
type LoadS3ApiConfigurationTestCase struct {
|
||||
pbIdent *iam_pb.Identity
|
||||
expectIdent *Identity
|
||||
}
|
||||
|
||||
func TestLoadS3ApiConfiguration(t *testing.T) {
|
||||
testCases := map[string]*LoadS3ApiConfigurationTestCase{
|
||||
"notSpecifyAccountId": {
|
||||
pbIdent: &iam_pb.Identity{
|
||||
Name: "notSpecifyAccountId",
|
||||
Actions: []string{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
Credentials: []*iam_pb.Credential{
|
||||
{
|
||||
AccessKey: "some_access_key1",
|
||||
SecretKey: "some_secret_key2",
|
||||
},
|
||||
},
|
||||
},
|
||||
expectIdent: &Identity{
|
||||
Name: "notSpecifyAccountId",
|
||||
AccountId: s3account.AccountAdmin.Id,
|
||||
Actions: []Action{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
Credentials: []*Credential{
|
||||
{
|
||||
AccessKey: "some_access_key1",
|
||||
SecretKey: "some_secret_key2",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"specifiedAccountID": {
|
||||
pbIdent: &iam_pb.Identity{
|
||||
Name: "specifiedAccountID",
|
||||
AccountId: "specifiedAccountID",
|
||||
Actions: []string{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
},
|
||||
expectIdent: &Identity{
|
||||
Name: "specifiedAccountID",
|
||||
AccountId: "specifiedAccountID",
|
||||
Actions: []Action{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
},
|
||||
},
|
||||
"anonymous": {
|
||||
pbIdent: &iam_pb.Identity{
|
||||
Name: "anonymous",
|
||||
Actions: []string{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
},
|
||||
expectIdent: &Identity{
|
||||
Name: "anonymous",
|
||||
AccountId: "anonymous",
|
||||
Actions: []Action{
|
||||
"Read",
|
||||
"Write",
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
config := &iam_pb.S3ApiConfiguration{
|
||||
Identities: make([]*iam_pb.Identity, 0),
|
||||
}
|
||||
for _, v := range testCases {
|
||||
config.Identities = append(config.Identities, v.pbIdent)
|
||||
}
|
||||
|
||||
iam := IdentityAccessManagement{}
|
||||
err := iam.loadS3ApiConfiguration(config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
for _, ident := range iam.identities {
|
||||
tc := testCases[ident.Name]
|
||||
if !reflect.DeepEqual(ident, tc.expectIdent) {
|
||||
t.Error("not expect")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"math"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
|
||||
entry, err := filer_pb.GetEntry(r.s3a, util.NewFullPath(r.s3a.option.BucketsPath, bucketName))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return buildBucketMetadata(r.s3a.accountManager, entry), nil
|
||||
}
|
||||
|
||||
type BucketMetaData struct {
|
||||
_ struct{} `type:"structure"`
|
||||
|
||||
Name string
|
||||
|
||||
//By default, when another AWS account uploads an object to S3 bucket,
|
||||
//that account (the object writer) owns the object, has access to it, and
|
||||
//can grant other users access to it through ACLs. You can use Object Ownership
|
||||
//to change this default behavior so that ACLs are disabled and you, as the
|
||||
//bucket owner, automatically own every object in your bucket.
|
||||
ObjectOwnership string
|
||||
|
||||
// Container for the bucket owner's display name and ID.
|
||||
Owner *s3.Owner `type:"structure"`
|
||||
|
||||
// A list of grants for access controls.
|
||||
Acl []*s3.Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"`
|
||||
}
|
||||
|
||||
type BucketRegistry struct {
|
||||
metadataCache map[string]*BucketMetaData
|
||||
metadataCacheLock sync.RWMutex
|
||||
|
||||
notFound map[string]struct{}
|
||||
notFoundLock sync.RWMutex
|
||||
s3a *S3ApiServer
|
||||
}
|
||||
|
||||
func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
|
||||
br := &BucketRegistry{
|
||||
metadataCache: make(map[string]*BucketMetaData),
|
||||
notFound: make(map[string]struct{}),
|
||||
s3a: s3a,
|
||||
}
|
||||
err := br.init()
|
||||
if err != nil {
|
||||
glog.Fatal("init bucket registry failed", err)
|
||||
return nil
|
||||
}
|
||||
return br
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) init() error {
|
||||
err := filer_pb.List(r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
|
||||
r.LoadBucketMetadata(entry)
|
||||
return nil
|
||||
}, "", false, math.MaxUint32)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
|
||||
bucketMetadata := buildBucketMetadata(r.s3a.accountManager, entry)
|
||||
r.metadataCacheLock.Lock()
|
||||
defer r.metadataCacheLock.Unlock()
|
||||
r.metadataCache[entry.Name] = bucketMetadata
|
||||
}
|
||||
|
||||
func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_pb.Entry) *BucketMetaData {
|
||||
entryJson, _ := json.Marshal(entry)
|
||||
glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
|
||||
bucketMetadata := &BucketMetaData{
|
||||
Name: entry.Name,
|
||||
|
||||
//Default ownership: OwnershipBucketOwnerEnforced, which means Acl is disabled
|
||||
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
|
||||
|
||||
// Default owner: `AccountAdmin`
|
||||
Owner: &s3.Owner{
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
},
|
||||
}
|
||||
if entry.Extended != nil {
|
||||
//ownership control
|
||||
ownership, ok := entry.Extended[s3_constants.ExtOwnershipKey]
|
||||
if ok {
|
||||
ownership := string(ownership)
|
||||
valid := s3_constants.ValidateOwnership(ownership)
|
||||
if valid {
|
||||
bucketMetadata.ObjectOwnership = ownership
|
||||
} else {
|
||||
glog.Warningf("Invalid ownership: %s, bucket: %s", ownership, bucketMetadata.Name)
|
||||
}
|
||||
}
|
||||
|
||||
//access control policy
|
||||
//owner
|
||||
acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey]
|
||||
if ok && len(acpOwnerBytes) > 0 {
|
||||
ownerAccountId := string(acpOwnerBytes)
|
||||
ownerAccountName, exists := accountManager.IdNameMapping[ownerAccountId]
|
||||
if !exists {
|
||||
glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name)
|
||||
} else {
|
||||
bucketMetadata.Owner = &s3.Owner{
|
||||
ID: &ownerAccountId,
|
||||
DisplayName: &ownerAccountName,
|
||||
}
|
||||
}
|
||||
}
|
||||
//grants
|
||||
acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey]
|
||||
if ok && len(acpGrantsBytes) > 0 {
|
||||
var grants []*s3.Grant
|
||||
err := json.Unmarshal(acpGrantsBytes, &grants)
|
||||
if err == nil {
|
||||
bucketMetadata.Acl = grants
|
||||
} else {
|
||||
glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
return bucketMetadata
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) {
|
||||
r.removeMetadataCache(entry.Name)
|
||||
r.unMarkNotFound(entry.Name)
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
|
||||
r.metadataCacheLock.RLock()
|
||||
bucketMetadata, ok := r.metadataCache[bucketName]
|
||||
r.metadataCacheLock.RUnlock()
|
||||
if ok {
|
||||
return bucketMetadata, s3err.ErrNone
|
||||
}
|
||||
|
||||
r.notFoundLock.RLock()
|
||||
_, ok = r.notFound[bucketName]
|
||||
r.notFoundLock.RUnlock()
|
||||
if ok {
|
||||
return nil, s3err.ErrNoSuchBucket
|
||||
}
|
||||
|
||||
bucketMetadata, errCode := r.LoadBucketMetadataFromFiler(bucketName)
|
||||
if errCode != s3err.ErrNone {
|
||||
return nil, errCode
|
||||
}
|
||||
|
||||
r.setMetadataCache(bucketMetadata)
|
||||
r.unMarkNotFound(bucketName)
|
||||
return bucketMetadata, s3err.ErrNone
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
|
||||
r.notFoundLock.Lock()
|
||||
defer r.notFoundLock.Unlock()
|
||||
|
||||
//check if already exists
|
||||
r.metadataCacheLock.RLock()
|
||||
bucketMetaData, ok := r.metadataCache[bucketName]
|
||||
r.metadataCacheLock.RUnlock()
|
||||
if ok {
|
||||
return bucketMetaData, s3err.ErrNone
|
||||
}
|
||||
|
||||
//if not exists, load from filer
|
||||
bucketMetadata, err := loadBucketMetadataFromFiler(r, bucketName)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
// The bucket doesn't actually exist and should no longer loaded from the filer
|
||||
r.notFound[bucketName] = struct{}{}
|
||||
return nil, s3err.ErrNoSuchBucket
|
||||
}
|
||||
return nil, s3err.ErrInternalError
|
||||
}
|
||||
return bucketMetadata, s3err.ErrNone
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) {
|
||||
r.metadataCacheLock.Lock()
|
||||
defer r.metadataCacheLock.Unlock()
|
||||
r.metadataCache[metadata.Name] = metadata
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) removeMetadataCache(bucket string) {
|
||||
r.metadataCacheLock.Lock()
|
||||
defer r.metadataCacheLock.Unlock()
|
||||
delete(r.metadataCache, bucket)
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) markNotFound(bucket string) {
|
||||
r.notFoundLock.Lock()
|
||||
defer r.notFoundLock.Unlock()
|
||||
r.notFound[bucket] = struct{}{}
|
||||
}
|
||||
|
||||
func (r *BucketRegistry) unMarkNotFound(bucket string) {
|
||||
r.notFoundLock.Lock()
|
||||
defer r.notFoundLock.Unlock()
|
||||
delete(r.notFound, bucket)
|
||||
}
|
||||
@@ -0,0 +1,231 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type BucketMetadataTestCase struct {
|
||||
filerEntry *filer_pb.Entry
|
||||
expectBucketMetadata *BucketMetaData
|
||||
}
|
||||
|
||||
var (
|
||||
//bad entry
|
||||
badEntry = &filer_pb.Entry{
|
||||
Name: "badEntry",
|
||||
}
|
||||
|
||||
//good entry
|
||||
goodEntryAcl, _ = json.Marshal(s3_constants.PublicRead)
|
||||
goodEntry = &filer_pb.Entry{
|
||||
Name: "entryWithValidAcp",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
|
||||
s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name),
|
||||
s3_constants.ExtAmzAclKey: goodEntryAcl,
|
||||
},
|
||||
}
|
||||
|
||||
//ownership is ""
|
||||
ownershipEmptyStr = &filer_pb.Entry{
|
||||
Name: "ownershipEmptyStr",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtOwnershipKey: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
//ownership valid
|
||||
ownershipValid = &filer_pb.Entry{
|
||||
Name: "ownershipValid",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
|
||||
},
|
||||
}
|
||||
|
||||
//owner is ""
|
||||
acpEmptyStr = &filer_pb.Entry{
|
||||
Name: "acpEmptyStr",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtAmzOwnerKey: []byte(""),
|
||||
},
|
||||
}
|
||||
|
||||
//owner not exists
|
||||
acpEmptyObject = &filer_pb.Entry{
|
||||
Name: "acpEmptyObject",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtAmzOwnerKey: []byte("xxxxx"),
|
||||
},
|
||||
}
|
||||
|
||||
//grants is nil
|
||||
acpOwnerNilAcp, _ = json.Marshal(make([]*s3.Grant, 0))
|
||||
acpOwnerNil = &filer_pb.Entry{
|
||||
Name: "acpOwnerNil",
|
||||
Extended: map[string][]byte{
|
||||
s3_constants.ExtAmzAclKey: acpOwnerNilAcp,
|
||||
},
|
||||
}
|
||||
|
||||
//load filer is
|
||||
loadFilerBucket = make(map[string]int, 1)
|
||||
//override `loadBucketMetadataFromFiler` to avoid really load from filer
|
||||
)
|
||||
|
||||
var tcs = []*BucketMetadataTestCase{
|
||||
{
|
||||
badEntry, &BucketMetaData{
|
||||
Name: badEntry.Name,
|
||||
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
goodEntry, &BucketMetaData{
|
||||
Name: goodEntry.Name,
|
||||
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: s3_constants.PublicRead,
|
||||
},
|
||||
},
|
||||
{
|
||||
ownershipEmptyStr, &BucketMetaData{
|
||||
Name: ownershipEmptyStr.Name,
|
||||
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
ownershipValid, &BucketMetaData{
|
||||
Name: ownershipValid.Name,
|
||||
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
acpEmptyStr, &BucketMetaData{
|
||||
Name: acpEmptyStr.Name,
|
||||
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
acpEmptyObject, &BucketMetaData{
|
||||
Name: acpEmptyObject.Name,
|
||||
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: nil,
|
||||
},
|
||||
},
|
||||
{
|
||||
acpOwnerNil, &BucketMetaData{
|
||||
Name: acpOwnerNil.Name,
|
||||
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
|
||||
Owner: &s3.Owner{
|
||||
DisplayName: &s3account.AccountAdmin.Name,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Acl: make([]*s3.Grant, 0),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestBuildBucketMetadata(t *testing.T) {
|
||||
accountManager := &s3account.AccountManager{
|
||||
IdNameMapping: map[string]string{
|
||||
s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
|
||||
s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
|
||||
},
|
||||
}
|
||||
for _, tc := range tcs {
|
||||
resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry)
|
||||
if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
|
||||
t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetBucketMetadata(t *testing.T) {
|
||||
loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
|
||||
time.Sleep(time.Second)
|
||||
loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
|
||||
return &BucketMetaData{
|
||||
Name: bucketName,
|
||||
}, nil
|
||||
}
|
||||
|
||||
br := &BucketRegistry{
|
||||
metadataCache: make(map[string]*BucketMetaData),
|
||||
notFound: make(map[string]struct{}),
|
||||
s3a: nil,
|
||||
}
|
||||
|
||||
//start 40 goroutine for
|
||||
var wg sync.WaitGroup
|
||||
closeCh := make(chan struct{})
|
||||
for i := 0; i < 40; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
outLoop:
|
||||
for {
|
||||
for j := 0; j < 5; j++ {
|
||||
select {
|
||||
case <-closeCh:
|
||||
break outLoop
|
||||
default:
|
||||
reqBucket := fmt.Sprintf("%c", 67+j)
|
||||
_, errCode := br.GetBucketMetadata(reqBucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
close(closeCh)
|
||||
t.Error("not expect")
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
}
|
||||
}()
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
close(closeCh)
|
||||
wg.Wait()
|
||||
|
||||
//Each bucket is loaded from the filer only once
|
||||
for bucketName, loadCount := range loadFilerBucket {
|
||||
if loadCount != 1 {
|
||||
t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
|
||||
return filer_pb.GetEntry(s3a, fullPath)
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
|
||||
updateEntryRequest := &filer_pb.UpdateEntryRequest{
|
||||
Directory: parentDirectoryPath,
|
||||
Entry: newEntry,
|
||||
}
|
||||
|
||||
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
|
||||
err := filer_pb.UpdateEntry(client, updateEntryRequest)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
||||
func objectKey(key *string) *string {
|
||||
if strings.HasPrefix(*key, "/") {
|
||||
t := (*key)[1:]
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
package s3_constants
|
||||
|
||||
import (
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
)
|
||||
|
||||
const (
|
||||
CannedAclPrivate = "private"
|
||||
CannedAclPublicRead = "public-read"
|
||||
CannedAclPublicReadWrite = "public-read-write"
|
||||
CannedAclAuthenticatedRead = "authenticated-read"
|
||||
CannedAclLogDeliveryWrite = "log-delivery-write"
|
||||
CannedAclBucketOwnerRead = "bucket-owner-read"
|
||||
CannedAclBucketOwnerFullControl = "bucket-owner-full-control"
|
||||
CannedAclAwsExecRead = "aws-exec-read"
|
||||
)
|
||||
|
||||
var (
|
||||
PublicRead = []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &GrantTypeGroup,
|
||||
URI: &GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &PermissionRead,
|
||||
},
|
||||
}
|
||||
|
||||
PublicReadWrite = []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &GrantTypeGroup,
|
||||
URI: &GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &PermissionRead,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &GrantTypeGroup,
|
||||
URI: &GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &PermissionWrite,
|
||||
},
|
||||
}
|
||||
|
||||
AuthenticatedRead = []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &GrantTypeGroup,
|
||||
URI: &GranteeGroupAuthenticatedUsers,
|
||||
},
|
||||
Permission: &PermissionRead,
|
||||
},
|
||||
}
|
||||
|
||||
LogDeliveryWrite = []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &GrantTypeGroup,
|
||||
URI: &GranteeGroupLogDelivery,
|
||||
},
|
||||
Permission: &PermissionWrite,
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,20 @@
|
||||
package s3_constants
|
||||
|
||||
//Amazon S3 predefined groups
|
||||
var (
|
||||
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
|
||||
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
|
||||
GranteeGroupLogDelivery = "http://acs.amazonaws.com/groups/s3/LogDelivery"
|
||||
)
|
||||
|
||||
func ValidateGroup(group string) bool {
|
||||
valid := true
|
||||
switch group {
|
||||
case GranteeGroupAllUsers:
|
||||
case GranteeGroupLogDelivery:
|
||||
case GranteeGroupAuthenticatedUsers:
|
||||
default:
|
||||
valid = false
|
||||
}
|
||||
return valid
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package s3_constants
|
||||
|
||||
var (
|
||||
GrantTypeCanonicalUser = "CanonicalUser"
|
||||
GrantTypeAmazonCustomerByEmail = "AmazonCustomerByEmail"
|
||||
GrantTypeGroup = "Group"
|
||||
)
|
||||
@@ -0,0 +1,18 @@
|
||||
package s3_constants
|
||||
|
||||
var (
|
||||
OwnershipBucketOwnerPreferred = "BucketOwnerPreferred"
|
||||
OwnershipObjectWriter = "ObjectWriter"
|
||||
OwnershipBucketOwnerEnforced = "BucketOwnerEnforced"
|
||||
|
||||
DefaultOwnershipForCreate = OwnershipObjectWriter
|
||||
DefaultOwnershipForExists = OwnershipBucketOwnerEnforced
|
||||
)
|
||||
|
||||
func ValidateOwnership(ownership string) bool {
|
||||
if ownership == "" || (ownership != OwnershipBucketOwnerPreferred && ownership != OwnershipObjectWriter && ownership != OwnershipBucketOwnerEnforced) {
|
||||
return false
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package s3_constants
|
||||
|
||||
var (
|
||||
PermissionFullControl = "FULL_CONTROL"
|
||||
PermissionRead = "READ"
|
||||
PermissionWrite = "WRITE"
|
||||
PermissionReadAcp = "READ_ACP"
|
||||
PermissionWriteAcp = "WRITE_ACP"
|
||||
)
|
||||
@@ -0,0 +1,7 @@
|
||||
package s3_constants
|
||||
|
||||
const (
|
||||
ExtAmzOwnerKey = "Seaweed-X-Amz-Owner"
|
||||
ExtAmzAclKey = "Seaweed-X-Amz-Acl"
|
||||
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
|
||||
)
|
||||
@@ -38,11 +38,20 @@ const (
|
||||
AmzTagCount = "x-amz-tagging-count"
|
||||
|
||||
X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key"
|
||||
|
||||
// S3 ACL headers
|
||||
AmzCannedAcl = "X-Amz-Acl"
|
||||
AmzAclFullControl = "X-Amz-Grant-Full-Control"
|
||||
AmzAclRead = "X-Amz-Grant-Read"
|
||||
AmzAclWrite = "X-Amz-Grant-Write"
|
||||
AmzAclReadAcp = "X-Amz-Grant-Read-Acp"
|
||||
AmzAclWriteAcp = "X-Amz-Grant-Write-Acp"
|
||||
)
|
||||
|
||||
// Non-Standard S3 HTTP request constants
|
||||
const (
|
||||
AmzIdentityId = "s3-identity-id"
|
||||
AmzAccountId = "s3-account-id"
|
||||
AmzAuthType = "s3-auth-type"
|
||||
AmzIsAdmin = "s3-is-admin" // only set to http request header as a context
|
||||
)
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package s3account
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"sync"
|
||||
)
|
||||
|
||||
//Predefined Accounts
|
||||
var (
|
||||
// AccountAdmin is used as the default account for IAM-Credentials access without Account configured
|
||||
AccountAdmin = Account{
|
||||
Name: "admin",
|
||||
EmailAddress: "admin@example.com",
|
||||
Id: "admin",
|
||||
}
|
||||
|
||||
// AccountAnonymous is used to represent the account for anonymous access
|
||||
AccountAnonymous = Account{
|
||||
Name: "anonymous",
|
||||
EmailAddress: "anonymous@example.com",
|
||||
Id: "anonymous",
|
||||
}
|
||||
)
|
||||
|
||||
//Account represents a system user, a system user can
|
||||
//configure multiple IAM-Users, IAM-Users can configure
|
||||
//permissions respectively, and each IAM-User can
|
||||
//configure multiple security credentials
|
||||
type Account struct {
|
||||
//Name is also used to display the "DisplayName" as the owner of the bucket or object
|
||||
Name string
|
||||
EmailAddress string
|
||||
|
||||
//Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects
|
||||
Id string
|
||||
}
|
||||
|
||||
type AccountManager struct {
|
||||
sync.Mutex
|
||||
filerClient filer_pb.FilerClient
|
||||
|
||||
IdNameMapping map[string]string
|
||||
EmailIdMapping map[string]string
|
||||
}
|
||||
|
||||
func NewAccountManager(filerClient filer_pb.FilerClient) *AccountManager {
|
||||
am := &AccountManager{
|
||||
filerClient: filerClient,
|
||||
IdNameMapping: make(map[string]string),
|
||||
EmailIdMapping: make(map[string]string),
|
||||
}
|
||||
am.initialize()
|
||||
return am
|
||||
}
|
||||
|
||||
func (am *AccountManager) GetAccountNameById(canonicalId string) string {
|
||||
return am.IdNameMapping[canonicalId]
|
||||
}
|
||||
|
||||
func (am *AccountManager) GetAccountIdByEmail(email string) string {
|
||||
return am.EmailIdMapping[email]
|
||||
}
|
||||
|
||||
func (am *AccountManager) initialize() {
|
||||
// load predefined Accounts
|
||||
for _, account := range []Account{AccountAdmin, AccountAnonymous} {
|
||||
am.IdNameMapping[account.Id] = account.Name
|
||||
am.EmailIdMapping[account.EmailAddress] = account.Id
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,509 @@
|
||||
package s3acl
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen
|
||||
func GetAccountId(r *http.Request) string {
|
||||
id := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if len(id) == 0 {
|
||||
return s3account.AccountAnonymous.Id
|
||||
} else {
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
// ExtractAcl extracts the acl from the request body, or from the header if request body is empty
|
||||
func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) {
|
||||
if r.Body != nil && r.Body != http.NoBody {
|
||||
defer util.CloseRequest(r)
|
||||
|
||||
var acp s3.AccessControlPolicy
|
||||
err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "")
|
||||
if err != nil || acp.Owner == nil || acp.Owner.ID == nil {
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
//owner should present && owner is immutable
|
||||
if *acp.Owner.ID != ownerId {
|
||||
glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId)
|
||||
return nil, s3err.ErrAccessDenied
|
||||
}
|
||||
|
||||
return ValidateAndTransferGrants(accountManager, acp.Grants)
|
||||
} else {
|
||||
_, grants, errCode = ParseAndValidateAclHeadersOrElseDefault(r, accountManager, ownership, bucketOwnerId, accountId, true)
|
||||
return grants, errCode
|
||||
}
|
||||
}
|
||||
|
||||
// ParseAndValidateAclHeadersOrElseDefault will callParseAndValidateAclHeaders to get Grants, if empty, it will return Grant that grant `accountId` with `FullControl` permission
|
||||
func ParseAndValidateAclHeadersOrElseDefault(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
|
||||
ownerId, grants, errCode = ParseAndValidateAclHeaders(r, accountManager, ownership, bucketOwnerId, accountId, putAcl)
|
||||
if errCode != s3err.ErrNone {
|
||||
return
|
||||
}
|
||||
if len(grants) == 0 {
|
||||
//if no acl(both customAcl and cannedAcl) specified, grant accountId(object writer) with full control permission
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ParseAndValidateAclHeaders parse and validate acl from header
|
||||
func ParseAndValidateAclHeaders(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
|
||||
ownerId, grants, errCode = ParseAclHeaders(r, ownership, bucketOwnerId, accountId, putAcl)
|
||||
if errCode != s3err.ErrNone {
|
||||
return
|
||||
}
|
||||
if len(grants) > 0 {
|
||||
grants, errCode = ValidateAndTransferGrants(accountManager, grants)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ParseAclHeaders parse acl headers
|
||||
// When `putAcl` is true, only `CannedAcl` is parsed, such as `PutBucketAcl` or `PutObjectAcl`
|
||||
// is requested, `CustomAcl` is parsed from the request body not from headers, and only if the
|
||||
// request body is empty, `CannedAcl` is parsed from the header, and will not parse `CustomAcl` from the header
|
||||
//
|
||||
// Since `CustomAcl` has higher priority, it will be parsed first; if `CustomAcl` does not exist, `CannedAcl` will be parsed
|
||||
func ParseAclHeaders(r *http.Request, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
|
||||
if !putAcl {
|
||||
errCode = ParseCustomAclHeaders(r, &grants)
|
||||
if errCode != s3err.ErrNone {
|
||||
return "", nil, errCode
|
||||
}
|
||||
}
|
||||
if len(grants) > 0 {
|
||||
return accountId, grants, s3err.ErrNone
|
||||
}
|
||||
|
||||
cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
|
||||
if len(cannedAcl) == 0 {
|
||||
return accountId, grants, s3err.ErrNone
|
||||
}
|
||||
|
||||
//if canned acl specified, parse cannedAcl (lower priority to custom acl)
|
||||
ownerId, grants, errCode = ParseCannedAclHeader(ownership, bucketOwnerId, accountId, cannedAcl, putAcl)
|
||||
if errCode != s3err.ErrNone {
|
||||
return "", nil, errCode
|
||||
}
|
||||
return ownerId, grants, errCode
|
||||
}
|
||||
|
||||
func ParseCustomAclHeaders(r *http.Request, grants *[]*s3.Grant) s3err.ErrorCode {
|
||||
customAclHeaders := []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp}
|
||||
var errCode s3err.ErrorCode
|
||||
for _, customAclHeader := range customAclHeaders {
|
||||
headerValue := r.Header.Get(customAclHeader)
|
||||
switch customAclHeader {
|
||||
case s3_constants.AmzAclRead:
|
||||
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, grants)
|
||||
case s3_constants.AmzAclWrite:
|
||||
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, grants)
|
||||
case s3_constants.AmzAclReadAcp:
|
||||
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, grants)
|
||||
case s3_constants.AmzAclWriteAcp:
|
||||
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, grants)
|
||||
case s3_constants.AmzAclFullControl:
|
||||
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, grants)
|
||||
}
|
||||
if errCode != s3err.ErrNone {
|
||||
return errCode
|
||||
}
|
||||
}
|
||||
return s3err.ErrNone
|
||||
}
|
||||
|
||||
func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s3err.ErrorCode {
|
||||
if len(headerValue) > 0 {
|
||||
split := strings.Split(headerValue, ", ")
|
||||
for _, grantStr := range split {
|
||||
kv := strings.Split(grantStr, "=")
|
||||
if len(kv) != 2 {
|
||||
return s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
switch kv[0] {
|
||||
case "id":
|
||||
var accountId string
|
||||
_ = json.Unmarshal([]byte(kv[1]), &accountId)
|
||||
*grants = append(*grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &permission,
|
||||
})
|
||||
case "emailAddress":
|
||||
var emailAddress string
|
||||
_ = json.Unmarshal([]byte(kv[1]), &emailAddress)
|
||||
*grants = append(*grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeAmazonCustomerByEmail,
|
||||
EmailAddress: &emailAddress,
|
||||
},
|
||||
Permission: &permission,
|
||||
})
|
||||
case "uri":
|
||||
var groupName string
|
||||
_ = json.Unmarshal([]byte(kv[1]), &groupName)
|
||||
*grants = append(*grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &groupName,
|
||||
},
|
||||
Permission: &permission,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
return s3err.ErrNone
|
||||
|
||||
}
|
||||
|
||||
func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl string, putAcl bool) (ownerId string, grants []*s3.Grant, err s3err.ErrorCode) {
|
||||
err = s3err.ErrNone
|
||||
ownerId = accountId
|
||||
|
||||
//objectWrite automatically has full control on current object
|
||||
objectWriterFullControl := &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
ID: &accountId,
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
}
|
||||
|
||||
switch cannedAcl {
|
||||
case s3_constants.CannedAclPrivate:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
case s3_constants.CannedAclPublicRead:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
grants = append(grants, s3_constants.PublicRead...)
|
||||
case s3_constants.CannedAclPublicReadWrite:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
grants = append(grants, s3_constants.PublicReadWrite...)
|
||||
case s3_constants.CannedAclAuthenticatedRead:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
grants = append(grants, s3_constants.AuthenticatedRead...)
|
||||
case s3_constants.CannedAclLogDeliveryWrite:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
grants = append(grants, s3_constants.LogDeliveryWrite...)
|
||||
case s3_constants.CannedAclBucketOwnerRead:
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
if bucketOwnerId != "" && bucketOwnerId != accountId {
|
||||
grants = append(grants,
|
||||
&s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &bucketOwnerId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
})
|
||||
}
|
||||
case s3_constants.CannedAclBucketOwnerFullControl:
|
||||
if bucketOwnerId != "" {
|
||||
// if set ownership to 'BucketOwnerPreferred' when upload object, the bucket owner will be the object owner
|
||||
if !putAcl && bucketOwnership == s3_constants.OwnershipBucketOwnerPreferred {
|
||||
ownerId = bucketOwnerId
|
||||
grants = append(grants,
|
||||
&s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &bucketOwnerId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
} else {
|
||||
grants = append(grants, objectWriterFullControl)
|
||||
if accountId != bucketOwnerId {
|
||||
grants = append(grants,
|
||||
&s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &bucketOwnerId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
case s3_constants.CannedAclAwsExecRead:
|
||||
err = s3err.ErrNotImplemented
|
||||
default:
|
||||
err = s3err.ErrInvalidRequest
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant
|
||||
func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) {
|
||||
var result []*s3.Grant
|
||||
for _, grant := range grants {
|
||||
grantee := grant.Grantee
|
||||
if grantee == nil || grantee.Type == nil {
|
||||
glog.Warning("invalid grantee! grantee or granteeType is nil")
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
|
||||
switch *grantee.Type {
|
||||
case s3_constants.GrantTypeGroup:
|
||||
if grantee.URI == nil {
|
||||
glog.Warning("invalid group grantee! group URI is nil")
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
ok := s3_constants.ValidateGroup(*grantee.URI)
|
||||
if !ok {
|
||||
glog.Warningf("invalid group grantee! group name[%s] is not valid", *grantee.URI)
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
result = append(result, grant)
|
||||
case s3_constants.GrantTypeCanonicalUser:
|
||||
if grantee.ID == nil {
|
||||
glog.Warning("invalid canonical grantee! account id is nil")
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
_, ok := accountManager.IdNameMapping[*grantee.ID]
|
||||
if !ok {
|
||||
glog.Warningf("invalid canonical grantee! account id[%s] is not exists", *grantee.ID)
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
result = append(result, grant)
|
||||
case s3_constants.GrantTypeAmazonCustomerByEmail:
|
||||
if grantee.EmailAddress == nil {
|
||||
glog.Warning("invalid email grantee! email address is nil")
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
accountId, ok := accountManager.EmailIdMapping[*grantee.EmailAddress]
|
||||
if !ok {
|
||||
glog.Warningf("invalid email grantee! email address[%s] is not exists", *grantee.EmailAddress)
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
result = append(result, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: grant.Permission,
|
||||
})
|
||||
default:
|
||||
return nil, s3err.ErrInvalidRequest
|
||||
}
|
||||
}
|
||||
return result, s3err.ErrNone
|
||||
}
|
||||
|
||||
// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission.
|
||||
func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
|
||||
// group grantee (AllUsers)
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &aclAction,
|
||||
})
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
|
||||
// canonical grantee (accountId)
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &aclAction,
|
||||
})
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
|
||||
// group grantee (AuthenticateUsers)
|
||||
if accountId != s3account.AccountAnonymous.Id {
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
|
||||
},
|
||||
Permission: &aclAction,
|
||||
})
|
||||
grants = append(grants, &s3.Grant{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
})
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func SetAcpOwnerHeader(r *http.Request, acpOwnerId string) {
|
||||
r.Header.Set(s3_constants.ExtAmzOwnerKey, acpOwnerId)
|
||||
}
|
||||
|
||||
func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string {
|
||||
ownerIdBytes, ok := entryExtended[s3_constants.ExtAmzOwnerKey]
|
||||
if ok && len(ownerIdBytes) > 0 {
|
||||
return string(ownerIdBytes)
|
||||
}
|
||||
return defaultOwner
|
||||
}
|
||||
|
||||
func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) {
|
||||
if len(acpGrants) > 0 {
|
||||
a, err := json.Marshal(acpGrants)
|
||||
if err == nil {
|
||||
r.Header.Set(s3_constants.ExtAmzAclKey, string(a))
|
||||
} else {
|
||||
glog.Warning("Marshal acp grants err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetAcpGrants return grants parsed from entry
|
||||
func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant {
|
||||
acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey]
|
||||
if ok && len(acpBytes) > 0 {
|
||||
var grants []*s3.Grant
|
||||
err := json.Unmarshal(acpBytes, &grants)
|
||||
if err == nil {
|
||||
return grants
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AssembleEntryWithAcp fill entry with owner and grants
|
||||
func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode {
|
||||
if objectEntry.Extended == nil {
|
||||
objectEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
|
||||
if len(objectOwner) > 0 {
|
||||
objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner)
|
||||
} else {
|
||||
delete(objectEntry.Extended, s3_constants.ExtAmzOwnerKey)
|
||||
}
|
||||
|
||||
if len(grants) > 0 {
|
||||
grantsBytes, err := json.Marshal(grants)
|
||||
if err != nil {
|
||||
glog.Warning("assemble acp to entry:", err)
|
||||
return s3err.ErrInvalidRequest
|
||||
}
|
||||
objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes
|
||||
} else {
|
||||
delete(objectEntry.Extended, s3_constants.ExtAmzAclKey)
|
||||
}
|
||||
|
||||
return s3err.ErrNone
|
||||
}
|
||||
|
||||
// GrantEquals Compare whether two Grants are equal in meaning, not completely
|
||||
// equal (compare Grantee.Type and the corresponding Value for equality, other
|
||||
// fields of Grantee are ignored)
|
||||
func GrantEquals(a, b *s3.Grant) bool {
|
||||
// grant
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
|
||||
if a == nil || b == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// grant.Permission
|
||||
if a.Permission != b.Permission {
|
||||
if a.Permission == nil || b.Permission == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if *a.Permission != *b.Permission {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// grant.Grantee
|
||||
ag := a.Grantee
|
||||
bg := b.Grantee
|
||||
if ag != bg {
|
||||
if ag == nil || bg == nil {
|
||||
return false
|
||||
}
|
||||
// grantee.Type
|
||||
if ag.Type != bg.Type {
|
||||
if ag.Type == nil || bg.Type == nil {
|
||||
return false
|
||||
}
|
||||
if *ag.Type != *bg.Type {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// value corresponding to granteeType
|
||||
if ag.Type != nil {
|
||||
switch *ag.Type {
|
||||
case s3_constants.GrantTypeGroup:
|
||||
if ag.URI != bg.URI {
|
||||
if ag.URI == nil || bg.URI == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if *ag.URI != *bg.URI {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case s3_constants.GrantTypeCanonicalUser:
|
||||
if ag.ID != bg.ID {
|
||||
if ag.ID == nil || bg.ID == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if *ag.ID != *bg.ID {
|
||||
return false
|
||||
}
|
||||
}
|
||||
case s3_constants.GrantTypeAmazonCustomerByEmail:
|
||||
if ag.EmailAddress != bg.EmailAddress {
|
||||
if ag.EmailAddress == nil || bg.EmailAddress == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if *ag.EmailAddress != *bg.EmailAddress {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
@@ -0,0 +1,706 @@
|
||||
package s3acl
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"github.com/aws/aws-sdk-go/service/s3"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"io"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var (
|
||||
accountManager = &s3account.AccountManager{
|
||||
IdNameMapping: map[string]string{
|
||||
s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
|
||||
s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
|
||||
"accountA": "accountA",
|
||||
"accountB": "accountB",
|
||||
},
|
||||
EmailIdMapping: map[string]string{
|
||||
s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id,
|
||||
s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id,
|
||||
"accountA@example.com": "accountA",
|
||||
"accountBexample.com": "accountB",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
func TestGetAccountId(t *testing.T) {
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
//case1
|
||||
//accountId: "admin"
|
||||
req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAdmin.Id)
|
||||
if GetAccountId(req) != s3account.AccountAdmin.Id {
|
||||
t.Fatal("expect accountId: admin")
|
||||
}
|
||||
|
||||
//case2
|
||||
//accountId: "anoymous"
|
||||
req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAnonymous.Id)
|
||||
if GetAccountId(req) != s3account.AccountAnonymous.Id {
|
||||
t.Fatal("expect accountId: anonymous")
|
||||
}
|
||||
|
||||
//case3
|
||||
//accountId is nil => "anonymous"
|
||||
req.Header.Del(s3_constants.AmzAccountId)
|
||||
if GetAccountId(req) != s3account.AccountAnonymous.Id {
|
||||
t.Fatal("expect accountId: anonymous")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractAcl(t *testing.T) {
|
||||
type Case struct {
|
||||
id int
|
||||
resultErrCode, expectErrCode s3err.ErrorCode
|
||||
resultGrants, expectGrants []*s3.Grant
|
||||
}
|
||||
testCases := make([]*Case, 0)
|
||||
accountAdminId := "admin"
|
||||
|
||||
{
|
||||
//case1 (good case)
|
||||
//parse acp from request body
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte(`
|
||||
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Owner>
|
||||
<ID>admin</ID>
|
||||
<DisplayName>admin</DisplayName>
|
||||
</Owner>
|
||||
<AccessControlList>
|
||||
<Grant>
|
||||
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
|
||||
<ID>admin</ID>
|
||||
</Grantee>
|
||||
<Permission>FULL_CONTROL</Permission>
|
||||
</Grant>
|
||||
<Grant>
|
||||
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
|
||||
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
|
||||
</Grantee>
|
||||
<Permission>FULL_CONTROL</Permission>
|
||||
</Grant>
|
||||
</AccessControlList>
|
||||
</AccessControlPolicy>
|
||||
`)))
|
||||
objectWriter := "accountA"
|
||||
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
|
||||
testCases = append(testCases, &Case{
|
||||
1,
|
||||
errCode, s3err.ErrNone,
|
||||
grants, []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountAdminId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
{
|
||||
//case2 (good case)
|
||||
//parse acp from header (cannedAcl)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
req.Body = nil
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
|
||||
objectWriter := "accountA"
|
||||
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
|
||||
testCases = append(testCases, &Case{
|
||||
2,
|
||||
errCode, s3err.ErrNone,
|
||||
grants, []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &objectWriter,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
{
|
||||
//case3 (bad case)
|
||||
//parse acp from request body (content is invalid)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf")))
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
|
||||
objectWriter := "accountA"
|
||||
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 3,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
|
||||
})
|
||||
}
|
||||
|
||||
//case4 (bad case)
|
||||
//parse acp from header (cannedAcl is invalid)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
req.Body = nil
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk")
|
||||
objectWriter := "accountA"
|
||||
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 4,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
|
||||
})
|
||||
|
||||
{
|
||||
//case5 (bad case)
|
||||
//parse acp from request body: owner is inconsistent
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte(`
|
||||
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Owner>
|
||||
<ID>admin</ID>
|
||||
<DisplayName>admin</DisplayName>
|
||||
</Owner>
|
||||
<AccessControlList>
|
||||
<Grant>
|
||||
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
|
||||
<ID>admin</ID>
|
||||
</Grantee>
|
||||
<Permission>FULL_CONTROL</Permission>
|
||||
</Grant>
|
||||
<Grant>
|
||||
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
|
||||
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
|
||||
</Grantee>
|
||||
<Permission>FULL_CONTROL</Permission>
|
||||
</Grant>
|
||||
</AccessControlList>
|
||||
</AccessControlPolicy>
|
||||
`)))
|
||||
objectWriter = "accountA"
|
||||
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 5,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied,
|
||||
})
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
if tc.resultErrCode != tc.expectErrCode {
|
||||
t.Fatalf("case[%d]: errorCode not expect", tc.id)
|
||||
}
|
||||
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
|
||||
t.Fatalf("case[%d]: grants not expect", tc.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseAndValidateAclHeaders(t *testing.T) {
|
||||
type Case struct {
|
||||
id int
|
||||
resultOwner, expectOwner string
|
||||
resultErrCode, expectErrCode s3err.ErrorCode
|
||||
resultGrants, expectGrants []*s3.Grant
|
||||
}
|
||||
testCases := make([]*Case, 0)
|
||||
bucketOwner := "admin"
|
||||
|
||||
{
|
||||
//case1 (good case)
|
||||
//parse custom acl
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`)
|
||||
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
1,
|
||||
ownerId, objectWriter,
|
||||
errCode, s3err.ErrNone,
|
||||
grants, []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &s3account.AccountAnonymous.Id,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
{
|
||||
//case2 (good case)
|
||||
//parse canned acl (ownership=ObjectWriter)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
|
||||
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
2,
|
||||
ownerId, objectWriter,
|
||||
errCode, s3err.ErrNone,
|
||||
grants, []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &objectWriter,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &bucketOwner,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
{
|
||||
//case3 (good case)
|
||||
//parse canned acl (ownership=OwnershipBucketOwnerPreferred)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
|
||||
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
3,
|
||||
ownerId, bucketOwner,
|
||||
errCode, s3err.ErrNone,
|
||||
grants, []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &bucketOwner,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
{
|
||||
//case4 (bad case)
|
||||
//parse custom acl (grantee id not exists)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`)
|
||||
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 4,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
|
||||
})
|
||||
}
|
||||
|
||||
{
|
||||
//case5 (bad case)
|
||||
//parse custom acl (invalid format)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`)
|
||||
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 5,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
|
||||
})
|
||||
}
|
||||
|
||||
{
|
||||
//case6 (bad case)
|
||||
//parse canned acl (invalid value)
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
objectWriter := "accountA"
|
||||
req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`)
|
||||
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
|
||||
testCases = append(testCases, &Case{
|
||||
id: 5,
|
||||
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
|
||||
})
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
if tc.expectErrCode != tc.resultErrCode {
|
||||
t.Errorf("case[%d]: errCode unexpect", tc.id)
|
||||
}
|
||||
if tc.resultOwner != tc.expectOwner {
|
||||
t.Errorf("case[%d]: ownerId unexpect", tc.id)
|
||||
}
|
||||
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
|
||||
t.Fatalf("case[%d]: grants not expect", tc.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func grantsEquals(a, b []*s3.Grant) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i, grant := range a {
|
||||
if !GrantEquals(grant, b[i]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func TestDetermineReqGrants(t *testing.T) {
|
||||
{
|
||||
//case1: request account is anonymous
|
||||
accountId := s3account.AccountAnonymous.Id
|
||||
reqPermission := s3_constants.PermissionRead
|
||||
|
||||
resultGrants := DetermineReqGrants(accountId, reqPermission)
|
||||
expectGrants := []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &reqPermission,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &reqPermission,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
}
|
||||
if !grantsEquals(resultGrants, expectGrants) {
|
||||
t.Fatalf("grants not expect")
|
||||
}
|
||||
}
|
||||
{
|
||||
//case2: request account is not anonymous (Iam authed)
|
||||
accountId := "accountX"
|
||||
reqPermission := s3_constants.PermissionRead
|
||||
|
||||
resultGrants := DetermineReqGrants(accountId, reqPermission)
|
||||
expectGrants := []*s3.Grant{
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &reqPermission,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &reqPermission,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeCanonicalUser,
|
||||
ID: &accountId,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
|
||||
},
|
||||
Permission: &reqPermission,
|
||||
},
|
||||
{
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
|
||||
},
|
||||
Permission: &s3_constants.PermissionFullControl,
|
||||
},
|
||||
}
|
||||
if !grantsEquals(resultGrants, expectGrants) {
|
||||
t.Fatalf("grants not expect")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAssembleEntryWithAcp(t *testing.T) {
|
||||
defaultOwner := "admin"
|
||||
|
||||
//case1
|
||||
//assemble with non-empty grants
|
||||
expectOwner := "accountS"
|
||||
expectGrants := []*s3.Grant{
|
||||
{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
},
|
||||
}
|
||||
entry := &filer_pb.Entry{}
|
||||
AssembleEntryWithAcp(entry, expectOwner, expectGrants)
|
||||
|
||||
resultOwner := GetAcpOwner(entry.Extended, defaultOwner)
|
||||
if resultOwner != expectOwner {
|
||||
t.Fatalf("owner not expect")
|
||||
}
|
||||
|
||||
resultGrants := GetAcpGrants(entry.Extended)
|
||||
if !grantsEquals(resultGrants, expectGrants) {
|
||||
t.Fatal("grants not expect")
|
||||
}
|
||||
|
||||
//case2
|
||||
//assemble with empty grants (override)
|
||||
AssembleEntryWithAcp(entry, "", nil)
|
||||
resultOwner = GetAcpOwner(entry.Extended, defaultOwner)
|
||||
if resultOwner != defaultOwner {
|
||||
t.Fatalf("owner not expect")
|
||||
}
|
||||
|
||||
resultGrants = GetAcpGrants(entry.Extended)
|
||||
if len(resultGrants) != 0 {
|
||||
t.Fatal("grants not expect")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestGrantEquals(t *testing.T) {
|
||||
testCases := map[bool]bool{
|
||||
GrantEquals(nil, nil): true,
|
||||
|
||||
GrantEquals(&s3.Grant{}, nil): false,
|
||||
|
||||
GrantEquals(&s3.Grant{}, &s3.Grant{}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
}, &s3.Grant{}): false,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{},
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{},
|
||||
}): false,
|
||||
|
||||
//type not present, compare other fields of grant is meaningless
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
EmailAddress: &s3account.AccountAdmin.EmailAddress,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
},
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionWrite,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}): false,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
}): true,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
},
|
||||
}): false,
|
||||
|
||||
GrantEquals(&s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}, &s3.Grant{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
}): true,
|
||||
}
|
||||
|
||||
for tc, expect := range testCases {
|
||||
if tc != expect {
|
||||
t.Fatal("TestGrantEquals not expect!")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetAcpOwnerHeader(t *testing.T) {
|
||||
ownerId := "accountZ"
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
SetAcpOwnerHeader(req, ownerId)
|
||||
|
||||
if req.Header.Get(s3_constants.ExtAmzOwnerKey) != ownerId {
|
||||
t.Fatalf("owner unexpect")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetAcpGrantsHeader(t *testing.T) {
|
||||
req := &http.Request{
|
||||
Header: make(map[string][]string),
|
||||
}
|
||||
grants := []*s3.Grant{
|
||||
{
|
||||
Permission: &s3_constants.PermissionRead,
|
||||
Grantee: &s3.Grantee{
|
||||
Type: &s3_constants.GrantTypeGroup,
|
||||
ID: &s3account.AccountAdmin.Id,
|
||||
URI: &s3_constants.GranteeGroupAllUsers,
|
||||
},
|
||||
},
|
||||
}
|
||||
SetAcpGrantsHeader(req, grants)
|
||||
|
||||
grantsJson, _ := json.Marshal(grants)
|
||||
if req.Header.Get(s3_constants.ExtAmzAclKey) != string(grantsJson) {
|
||||
t.Fatalf("owner unexpect")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package s3api
|
||||
|
||||
import (
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func getAccountId(r *http.Request) string {
|
||||
id := r.Header.Get(s3_constants.AmzAccountId)
|
||||
if len(id) == 0 {
|
||||
return s3account.AccountAnonymous.Id
|
||||
} else {
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
|
||||
metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
return errCode
|
||||
}
|
||||
accountId := getAccountId(r)
|
||||
if accountId == s3account.AccountAdmin.Id || accountId == *metadata.Owner.ID {
|
||||
return s3err.ErrNone
|
||||
}
|
||||
return s3err.ErrAccessDenied
|
||||
}
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||
"math"
|
||||
"net/http"
|
||||
"time"
|
||||
@@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.
|
||||
func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
|
||||
}
|
||||
|
||||
// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
|
||||
func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
|
||||
|
||||
errCode := s3a.checkAccessByOwnership(r, bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
if r.Body == nil || r.Body == http.NoBody {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var v s3.OwnershipControls
|
||||
defer util.CloseRequest(r)
|
||||
|
||||
err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
|
||||
return
|
||||
}
|
||||
|
||||
if len(v.Rules) != 1 {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
|
||||
return
|
||||
}
|
||||
|
||||
printOwnership := true
|
||||
ownership := *v.Rules[0].ObjectOwnership
|
||||
switch ownership {
|
||||
case s3_constants.OwnershipObjectWriter:
|
||||
case s3_constants.OwnershipBucketOwnerPreferred:
|
||||
case s3_constants.OwnershipBucketOwnerEnforced:
|
||||
printOwnership = false
|
||||
default:
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
|
||||
return
|
||||
}
|
||||
|
||||
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
|
||||
if !ok || string(oldOwnership) != ownership {
|
||||
if bucketEntry.Extended == nil {
|
||||
bucketEntry.Extended = make(map[string][]byte)
|
||||
}
|
||||
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
|
||||
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if printOwnership {
|
||||
result := &s3.PutBucketOwnershipControlsInput{
|
||||
OwnershipControls: &v,
|
||||
}
|
||||
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
|
||||
} else {
|
||||
writeSuccessResponseEmpty(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
|
||||
func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
|
||||
|
||||
errCode := s3a.checkAccessByOwnership(r, bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
|
||||
if !ok {
|
||||
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
|
||||
return
|
||||
}
|
||||
ownership := string(v)
|
||||
|
||||
result := &s3.PutBucketOwnershipControlsInput{
|
||||
OwnershipControls: &s3.OwnershipControls{
|
||||
Rules: []*s3.OwnershipControlsRule{
|
||||
{
|
||||
ObjectOwnership: &ownership,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
|
||||
}
|
||||
|
||||
// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
|
||||
func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
|
||||
bucket, _ := s3_constants.GetBucketAndObject(r)
|
||||
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
|
||||
|
||||
errCode := s3a.checkAccessByOwnership(r, bucket)
|
||||
if errCode != s3err.ErrNone {
|
||||
s3err.WriteErrorResponse(w, r, errCode)
|
||||
return
|
||||
}
|
||||
|
||||
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
|
||||
if err != nil {
|
||||
if err == filer_pb.ErrNotFound {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
|
||||
return
|
||||
}
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
|
||||
if !ok {
|
||||
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
|
||||
return
|
||||
}
|
||||
|
||||
delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
|
||||
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
|
||||
if err != nil {
|
||||
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
|
||||
return
|
||||
}
|
||||
|
||||
emptyOwnershipControls := &s3.OwnershipControls{
|
||||
Rules: []*s3.OwnershipControlsRule{},
|
||||
}
|
||||
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
|
||||
}
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -40,6 +41,8 @@ type S3ApiServer struct {
|
||||
randomClientId int32
|
||||
filerGuard *security.Guard
|
||||
client *http.Client
|
||||
accountManager *s3account.AccountManager
|
||||
bucketRegistry *BucketRegistry
|
||||
}
|
||||
|
||||
func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
|
||||
@@ -59,6 +62,8 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
|
||||
filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
|
||||
cb: NewCircuitBreaker(option),
|
||||
}
|
||||
s3ApiServer.accountManager = s3account.NewAccountManager(s3ApiServer)
|
||||
s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
|
||||
if option.LocalFilerSocket == "" {
|
||||
s3ApiServer.client = &http.Client{Transport: &http.Transport{
|
||||
MaxIdleConns: 1024,
|
||||
@@ -76,7 +81,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
|
||||
|
||||
s3ApiServer.registerRouter(router)
|
||||
|
||||
go s3ApiServer.subscribeMetaEvents("s3", filer.DirectoryEtcRoot, time.Now().UnixNano())
|
||||
go s3ApiServer.subscribeMetaEvents("s3", time.Now().UnixNano(), filer.DirectoryEtcRoot, []string{option.BucketsPath})
|
||||
return s3ApiServer, nil
|
||||
}
|
||||
|
||||
@@ -212,6 +217,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
|
||||
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
|
||||
|
||||
// buckets with query
|
||||
// PutBucketOwnershipControls
|
||||
bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
|
||||
|
||||
//GetBucketOwnershipControls
|
||||
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
|
||||
|
||||
//DeleteBucketOwnershipControls
|
||||
bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
|
||||
|
||||
// raw buckets
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"net/http"
|
||||
@@ -19,6 +20,16 @@ const (
|
||||
MimeXML mimeType = "application/xml"
|
||||
)
|
||||
|
||||
func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) {
|
||||
var bytesBuffer bytes.Buffer
|
||||
err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer))
|
||||
if err != nil {
|
||||
WriteErrorResponse(w, r, ErrInternalError)
|
||||
return
|
||||
}
|
||||
WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML)
|
||||
}
|
||||
|
||||
func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
|
||||
WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
|
||||
}
|
||||
|
||||
@@ -107,6 +107,8 @@ const (
|
||||
|
||||
ErrTooManyRequest
|
||||
ErrRequestBytesExceed
|
||||
|
||||
OwnershipControlsNotFoundError
|
||||
)
|
||||
|
||||
// error code to APIError structure, these fields carry respective
|
||||
@@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
|
||||
Description: "Simultaneous request bytes exceed limitations",
|
||||
HTTPStatusCode: http.StatusTooManyRequests,
|
||||
},
|
||||
|
||||
OwnershipControlsNotFoundError: {
|
||||
Code: "OwnershipControlsNotFoundError",
|
||||
Description: "The bucket ownership controls were not found",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
}
|
||||
|
||||
// GetAPIError provides API Error for input API error code.
|
||||
|
||||
@@ -33,6 +33,12 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
|
||||
recorder := NewStatusResponseWriter(w)
|
||||
start := time.Now()
|
||||
f(recorder, r)
|
||||
if recorder.Status == http.StatusForbidden {
|
||||
if m, _ := stats_collect.S3RequestCounter.GetMetricWithLabelValues(
|
||||
action, strconv.Itoa(http.StatusOK), bucket); m == nil {
|
||||
bucket = ""
|
||||
}
|
||||
}
|
||||
stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
|
||||
stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
|
||||
}
|
||||
|
||||
@@ -16,6 +16,15 @@ import (
|
||||
|
||||
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "*")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
||||
}
|
||||
|
||||
if r.Method == "OPTIONS" {
|
||||
OptionsHandler(w, r, false)
|
||||
return
|
||||
@@ -45,10 +54,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}()
|
||||
|
||||
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
fs.GetOrHeadHandler(w, r)
|
||||
@@ -92,6 +98,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
start := time.Now()
|
||||
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
|
||||
defer func() {
|
||||
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
|
||||
@@ -108,10 +121,7 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
|
||||
if r.Header.Get("Origin") != "" {
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
fs.GetOrHeadHandler(w, r)
|
||||
@@ -122,11 +132,14 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) {
|
||||
if isReadOnly {
|
||||
w.Header().Add("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
} else {
|
||||
w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "*")
|
||||
}
|
||||
w.Header().Add("Access-Control-Allow-Headers", "*")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Allow-Headers", "*")
|
||||
w.Header().Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
// maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied
|
||||
|
||||
@@ -107,9 +107,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query()
|
||||
|
||||
if entry.IsDirectory() {
|
||||
if fs.option.DisableDirListing {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
if query.Get("metadata") == "true" {
|
||||
writeJsonQuiet(w, r, http.StatusOK, entry)
|
||||
return
|
||||
}
|
||||
if entry.Attr.Mime == "" {
|
||||
@@ -125,7 +131,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
|
||||
query := r.URL.Query()
|
||||
if query.Get("metadata") == "true" {
|
||||
if query.Get("resolveManifest") == "true" {
|
||||
if entry.Chunks, _, err = filer.ResolveChunkManifest(
|
||||
|
||||
@@ -3,6 +3,7 @@ package weed_server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
//"github.com/seaweedfs/seaweedfs/weed/s3api"
|
||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -374,6 +375,18 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
|
||||
}
|
||||
}
|
||||
|
||||
//acp-owner
|
||||
acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey)
|
||||
if len(acpOwner) > 0 {
|
||||
metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner)
|
||||
}
|
||||
|
||||
//acp-grants
|
||||
acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey)
|
||||
if len(acpOwner) > 0 {
|
||||
metadata[s3_constants.ExtAmzAclKey] = []byte(acpGrants)
|
||||
}
|
||||
|
||||
return
|
||||
|
||||
}
|
||||
@@ -307,6 +307,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
|
||||
case <-ticker.C:
|
||||
if !ms.Topo.IsLeader() {
|
||||
stats.MasterRaftIsleader.Set(0)
|
||||
stats.MasterAdminLock.Reset()
|
||||
stats.MasterReplicaPlacementMismatch.Reset()
|
||||
return ms.informNewLeader(stream)
|
||||
} else {
|
||||
stats.MasterRaftIsleader.Set(1)
|
||||
|
||||
@@ -3,6 +3,7 @@ package weed_server
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -112,11 +113,13 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t
|
||||
lastClient: clientName,
|
||||
}
|
||||
locks.locks[lockName] = lock
|
||||
stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
|
||||
return lock.accessLockTime, lock.accessSecret
|
||||
}
|
||||
|
||||
func (locks *AdminLocks) deleteLock(lockName string) {
|
||||
locks.Lock()
|
||||
stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0)
|
||||
defer locks.Unlock()
|
||||
delete(locks.locks, lockName)
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
|
||||
}
|
||||
|
||||
servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
|
||||
_, leaderId := ms.Topo.HashicorpRaft.LeaderWithID()
|
||||
ms.Topo.RaftServerAccessLock.RUnlock()
|
||||
|
||||
for _, server := range servers {
|
||||
@@ -27,6 +28,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
|
||||
Id: string(server.ID),
|
||||
Address: string(server.Address),
|
||||
Suffrage: server.Suffrage.String(),
|
||||
IsLeader: server.ID == leaderId,
|
||||
})
|
||||
}
|
||||
return resp, nil
|
||||
|
||||
@@ -174,39 +174,45 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
|
||||
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
|
||||
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
|
||||
if ms.Topo.RaftServer.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
|
||||
glog.V(0).Infof("[%s] %s becomes leader.", ms.Topo.RaftServer.Name(), ms.Topo.RaftServer.Leader())
|
||||
}
|
||||
})
|
||||
raftServerName = ms.Topo.RaftServer.Name()
|
||||
raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name())
|
||||
} else if raftServer.RaftHashicorp != nil {
|
||||
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
|
||||
leaderCh := raftServer.RaftHashicorp.LeaderCh()
|
||||
prevLeader := ms.Topo.HashicorpRaft.Leader()
|
||||
prevLeader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
||||
raftServerName = ms.Topo.HashicorpRaft.String()
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case isLeader := <-leaderCh:
|
||||
leader := ms.Topo.HashicorpRaft.Leader()
|
||||
ms.Topo.RaftServerAccessLock.RLock()
|
||||
leader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
||||
ms.Topo.RaftServerAccessLock.RUnlock()
|
||||
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
|
||||
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
|
||||
prevLeader = leader
|
||||
}
|
||||
}
|
||||
}()
|
||||
raftServerName = ms.Topo.HashicorpRaft.String()
|
||||
}
|
||||
ms.Topo.RaftServerAccessLock.Unlock()
|
||||
|
||||
if ms.Topo.IsLeader() {
|
||||
glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
|
||||
glog.V(0).Infof("%s I am the leader!", raftServerName)
|
||||
} else {
|
||||
var raftServerLeader string
|
||||
ms.Topo.RaftServerAccessLock.RLock()
|
||||
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
|
||||
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
|
||||
glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
|
||||
if ms.Topo.RaftServer != nil {
|
||||
raftServerLeader = ms.Topo.RaftServer.Leader()
|
||||
} else if ms.Topo.HashicorpRaft != nil {
|
||||
raftServerName = ms.Topo.HashicorpRaft.String()
|
||||
raftServerLeaderAddr, _ := ms.Topo.HashicorpRaft.LeaderWithID()
|
||||
raftServerLeader = string(raftServerLeaderAddr)
|
||||
}
|
||||
ms.Topo.RaftServerAccessLock.RUnlock()
|
||||
glog.V(0).Infof("%s %s - is the leader.", raftServerName, raftServerLeader)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -388,8 +394,17 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF
|
||||
} else {
|
||||
glog.V(0).Infof("master %s successfully responded to ping", peerName)
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *MasterServer) Shutdown() {
|
||||
if ms.Topo == nil || ms.Topo.HashicorpRaft == nil {
|
||||
return
|
||||
}
|
||||
if ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
|
||||
ms.Topo.HashicorpRaft.LeadershipTransfer()
|
||||
}
|
||||
ms.Topo.HashicorpRaft.Shutdown()
|
||||
}
|
||||
@@ -6,10 +6,13 @@ package weed_server
|
||||
import (
|
||||
"fmt"
|
||||
transport "github.com/Jille/raft-grpc-transport"
|
||||
"github.com/armon/go-metrics"
|
||||
"github.com/armon/go-metrics/prometheus"
|
||||
"github.com/hashicorp/raft"
|
||||
boltdb "github.com/hashicorp/raft-boltdb/v2"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||
"google.golang.org/grpc"
|
||||
"math/rand"
|
||||
"os"
|
||||
@@ -182,5 +185,18 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Configure a prometheus sink as the raft metrics sink
|
||||
if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
|
||||
Registerer: stats.Gather,
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("NewPrometheusSink: %v", err)
|
||||
} else {
|
||||
metricsConf := metrics.DefaultConfig(stats.Namespace)
|
||||
metricsConf.EnableRuntimeMetrics = false
|
||||
if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
|
||||
return nil, fmt.Errorf("metrics.NewGlobal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
Loaded 100 of 140 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user