Compare commits

...
94 Commits
Author SHA1 Message Date
chrislu 758d70bdc0 3.32 2022-10-16 20:43:37 -07:00
chrislu de286fe662 shell: volume.move handles volume moved to cloud tier
fix https://github.com/seaweedfs/seaweedfs/issues/3803
2022-10-16 17:52:22 -07:00
chrislu 52e0a88a15 shell: volume.tier.upload clean up replicated copies
fix https://github.com/seaweedfs/seaweedfs/issues/3804
2022-10-16 17:02:56 -07:00
Ryan Russell 949607fe90 refactor(compose)!: upgrade to v2 closes #3699 (#3705)
* docs(compose): Update readme for compose v2 upgrade

Signed-off-by: Ryan Russell <git@ryanrussell.org>

* refactor(compose): Upgrade `Makefile` for compose v2

Signed-off-by: Ryan Russell <git@ryanrussell.org>

* refactor(compose): Upgrade yml from version 2 -> 3.9

Signed-off-by: Ryan Russell <git@ryanrussell.org>

Signed-off-by: Ryan Russell <git@ryanrussell.org>
2022-10-16 14:02:33 -07:00
Konstantin Lebedev e20f0dbd2d avoid data race of TraverseBfs (#3856)
* avoid data race of TraverseBfs

* close is enough
avoid panic
I1014 12:29:59.207120 volume_loading.go:131 loading sorted db /tmp/sw/test2_19.sdx error: unexpected file /tmp/sw/test2_19.idx size: 255
I1014 12:29:59.207125 volume_loading.go:119 open to write file /tmp/sw/test4_26.idx
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x5260a4c]

goroutine 287 [running]:
github.com/seaweedfs/seaweedfs/weed/storage.(*SortedFileNeedleMap).Close(0x0)
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/needle_map_sorted_file.go:97 +0x2c
github.com/seaweedfs/seaweedfs/weed/storage.(*Volume).load.func1()
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/volume_loading.go:32 +0x8e
github.com/seaweedfs/seaweedfs/weed/storage.(*Volume).load(0xc001b36280, 0x1, 0x1, 0x0, 0x69228c0?)
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/volume_loading.go:205 +0x256c
github.com/seaweedfs/seaweedfs/weed/storage.NewVolume({0x7ffeefbff6e0, 0x7}, {0x7ffeefbff6e0, 0x7}, {0xc0009a9284, 0x5}, 0x13, 0x0, 0x0, 0x0, ...)
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/volume.go:62 +0x30f
github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).loadExistingVolume(0xc0006f40c0, {0x846c8d0, 0xc0009ce600}, 0x0?, 0x1)
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:161 +0x4da
github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).concurrentLoadingVolumes.func2()
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:201 +0xf9
created by github.com/seaweedfs/seaweedfs/weed/storage.(*DiskLocation).concurrentLoadingVolumes
        /Users/tochka/GolandProjects/seaweedfs/weed/storage/disk_location.go:198 +0x150
2022-10-16 11:30:41 -07:00
Konstantin Lebedev ee38ab8581 avoid data race of LogBuffer isStopping (#3859) 2022-10-16 11:28:56 -07:00
Konstantin Lebedev 7836f7574e [volume.fsck] hotfix apply purging and add option verifyNeedle #3860 (#3861)
* fix apply purging and add verifyNeedle

* common readSourceNeedleBlob

* use consts
2022-10-15 20:38:46 -07:00
Konstantin Lebedev f476cf3403 fix volume_server.proto conflict (#3858)
fix conflict

https://github.com/seaweedfs/seaweedfs/pull/3841
2022-10-15 20:01:07 -07:00
chrislu 5c2eb602c9 Revert "mount: remove mount directory from generating cache directory id"
This reverts commit 109dc7fdf6.
2022-10-14 22:24:14 -07:00
Konstantin Lebedev 2f72103c83 avoid load volume file with BytesOffset mismatch (#3841)
* avoid load volume file with BytesOffset mismatch

https://github.com/seaweedfs/seaweedfs/issues/2966

* set BytesOffset if has not VolumeInfoFile

* typos fail => failed

* exit if bytesOffset mismatch
2022-10-14 00:18:09 -07:00
Konstantin Lebedev f19c9e3d9d Volume fsck by volume (#3851)
* refactor

* refactor args verbose and writer

* refactor readFilerFileIdFile

* fix filter by collectMtime

* skip system log collection
2022-10-13 23:30:30 -07:00
chrislu 9c8678ded9 ensure monotonic n.AppendAtNs
fix https://github.com/seaweedfs/seaweedfs/issues/3852
2022-10-13 23:15:00 -07:00
chrislu e05637c42c Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2022-10-13 23:10:49 -07:00
chrislu dcd0743a35 remove unused ReadNeedleBlobRequest.needle_id
fix https://github.com/seaweedfs/seaweedfs/issues/3853
2022-10-13 23:10:46 -07:00
LHHDZ a5b867af69 don't skip empty value at AssembleEntryWithAcp (#3855)
* add acl helper functionalities

Signed-off-by: changlin.shi <changlin.shi@ly.com>

* add tests

Signed-off-by: changlin.shi <changlin.shi@ly.com>

* remove 0 when create map

Signed-off-by: changlin.shi <changlin.shi@ly.com>

* delete when empty at `AssembleEntryWithAcp`

`PutBucketAcl/PutObjectAcl` allow request with empty grants, `AssembleEntryWithAcp`  shouldn't skip empty value

Signed-off-by: changlin.shi <changlin.shi@ly.com>

Signed-off-by: changlin.shi <changlin.shi@ly.com>
2022-10-13 22:50:44 -07:00
Konstantin Lebedev 1f7e52c63e vacuum metrics and force sync dst files (#3832) 2022-10-13 00:51:20 -07:00
chrislu f5d4952d73 filer: redis store reduce from 2 redis operations to 1 for updates. 2022-10-12 23:50:09 -07:00
Guo Lei f95c25e113 types packages is imported more than onece (#3838) 2022-10-12 22:59:07 -07:00
chrislu 9a339a9cfb default hasSlowRead to true 2022-10-12 21:15:10 -07:00
famosss cacc3e883b volume server:set the default value of "hasSlowRead" to true (#3710)
* simplify a bit

* feat: volume: add "readBufSize" option to customize read optimization

* refactor : redbufSIze -> readBufferSize

* simplify a bit

* simplify a bit

* volume server:set the default value of "hasSlowRead" to true
2022-10-12 21:13:26 -07:00
LHHDZ bf5e45b66a add acl helper functionalities (#3831) 2022-10-12 20:15:16 -07:00
Konstantin Lebedev 401315f337 master fix interruption through ctrl+c (#3834) 2022-10-12 07:18:40 -07:00
zemul a05725aea6 filer: get directory metadata (#3833) 2022-10-12 07:14:49 -07:00
chrislu e55076c46f cloud tier: add retry when copying data file
fix https://github.com/seaweedfs/seaweedfs/issues/3828
2022-10-12 00:38:32 -07:00
chrislu cea73ac008 serialize sqlite operations
fix https://github.com/seaweedfs/seaweedfs/issues/3827
2022-10-12 00:03:27 -07:00
chrislu dff85e9c71 fix error handling 2022-10-12 00:03:14 -07:00
chrislu 8aec786a6d Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2022-10-11 21:58:19 -07:00
chrislu 0d817bc347 fix invalid memory address or nil pointer dereference on filer.sync
fix https://github.com/seaweedfs/seaweedfs/issues/3826
2022-10-11 21:58:17 -07:00
Guo Lei 84c401e693 Optimiz leveldb metric (#3830)
* optimiz updating mapmetric for leveldb

* import loading leveldb

* add comments
2022-10-11 21:13:25 -07:00
LHHDZ d21e2f523d split ExtAcpKey to ExtAmzOwnerKey and ExtAmzAclKey to avoid unn… (#3824)
split `ExtAcpKey` to `ExtAmzOwnerKey` and `ExtAmzAclKey` to avoid unnecessary `json.Unmarshal()` call

Signed-off-by: changlin.shi <changlin.shi@ly.com>

Signed-off-by: changlin.shi <changlin.shi@ly.com>
2022-10-11 20:14:14 -07:00
livehlandhuang.lin c34f04b35b fix bug : read error when position after 2147483647 (#3829)
Co-authored-by: huang.lin <hh@chaintool.ai>
2022-10-11 17:50:12 -07:00
dependabot[bot] f6c8720ded build(deps): bump google.golang.org/grpc from 1.49.0 to 1.50.0 (#3819)
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.49.0 to 1.50.0.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.49.0...v1.50.0)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 12:15:51 -07:00
dependabot[bot] 50ed9456ff build(deps): bump github.com/hashicorp/raft from 1.3.10 to 1.3.11 (#3821)
Bumps [github.com/hashicorp/raft](https://github.com/hashicorp/raft) from 1.3.10 to 1.3.11.
- [Release notes](https://github.com/hashicorp/raft/releases)
- [Changelog](https://github.com/hashicorp/raft/blob/main/CHANGELOG.md)
- [Commits](https://github.com/hashicorp/raft/compare/v1.3.10...v1.3.11)

---
updated-dependencies:
- dependency-name: github.com/hashicorp/raft
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 12:15:29 -07:00
dependabot[bot] 0aa947913a build(deps): bump github.com/arangodb/go-driver from 1.3.3 to 1.4.0 (#3823)
Bumps [github.com/arangodb/go-driver](https://github.com/arangodb/go-driver) from 1.3.3 to 1.4.0.
- [Release notes](https://github.com/arangodb/go-driver/releases)
- [Changelog](https://github.com/arangodb/go-driver/blob/master/CHANGELOG.md)
- [Commits](https://github.com/arangodb/go-driver/compare/v1.3.3...v1.4.0)

---
updated-dependencies:
- dependency-name: github.com/arangodb/go-driver
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 12:15:19 -07:00
dependabot[bot] 358d9fb109 build(deps): bump github.com/armon/go-metrics from 0.3.10 to 0.4.1 (#3822)
Bumps [github.com/armon/go-metrics](https://github.com/armon/go-metrics) from 0.3.10 to 0.4.1.
- [Release notes](https://github.com/armon/go-metrics/releases)
- [Commits](https://github.com/armon/go-metrics/compare/v0.3.10...v0.4.1)

---
updated-dependencies:
- dependency-name: github.com/armon/go-metrics
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 12:01:49 -07:00
dependabot[bot] 3fbf68de12 build(deps): bump github.com/Shopify/sarama from 1.37.0 to 1.37.2 (#3818)
Bumps [github.com/Shopify/sarama](https://github.com/Shopify/sarama) from 1.37.0 to 1.37.2.
- [Release notes](https://github.com/Shopify/sarama/releases)
- [Changelog](https://github.com/Shopify/sarama/blob/main/CHANGELOG.md)
- [Commits](https://github.com/Shopify/sarama/compare/v1.37.0...v1.37.2)

---
updated-dependencies:
- dependency-name: github.com/Shopify/sarama
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 12:01:29 -07:00
dependabot[bot] c4fa148ee6 build(deps): bump actions/checkout from 3.0.2 to 3.1.0 (#3815)
Bumps [actions/checkout](https://github.com/actions/checkout) from 3.0.2 to 3.1.0.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/2541b1294d2704b0964813337f33b291d3f8596b...93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 11:47:15 -07:00
dependabot[bot] 6bbc1d4026 build(deps): bump github.com/klauspost/reedsolomon from 1.11.0 to 1.11.1 (#3816)
Bumps [github.com/klauspost/reedsolomon](https://github.com/klauspost/reedsolomon) from 1.11.0 to 1.11.1.
- [Release notes](https://github.com/klauspost/reedsolomon/releases)
- [Commits](https://github.com/klauspost/reedsolomon/compare/v1.11.0...v1.11.1)

---
updated-dependencies:
- dependency-name: github.com/klauspost/reedsolomon
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 11:47:07 -07:00
dependabot[bot] 43175e7fe8 build(deps): bump go.mongodb.org/mongo-driver from 1.10.2 to 1.10.3 (#3817)
Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.10.2 to 1.10.3.
- [Release notes](https://github.com/mongodb/mongo-go-driver/releases)
- [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.10.2...v1.10.3)

---
updated-dependencies:
- dependency-name: go.mongodb.org/mongo-driver
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 11:46:58 -07:00
dependabot[bot] 7617567c02 build(deps): bump github.com/aws/aws-sdk-go from 1.44.110 to 1.44.114 (#3820)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.110 to 1.44.114.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.110...v1.44.114)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-10 11:46:38 -07:00
Konstantin Lebedev 5b28c3f728 revert disabling FSync for non Mac (#3814) 2022-10-10 07:28:02 -07:00
LHHDZ 3550692afc change s3_account.go package to avoid cycle dependency (#3813) 2022-10-10 06:44:29 -07:00
chrislu 0711870f43 3.31 2022-10-10 00:15:30 -07:00
chrislu 109dc7fdf6 mount: remove mount directory from generating cache directory id
fix https://github.com/seaweedfs/seaweedfs/discussions/3811
2022-10-10 00:12:10 -07:00
chrislu fc54afdba9 tests add parsing ec shard info 2022-10-09 22:47:55 -07:00
chrislu 0623bf582e include ec shard for capacityByFreeVolumeCount 2022-10-09 22:31:30 -07:00
chrislu 98dc1e5c15 move volume: find target volume server by exiting/max ratio 2022-10-09 22:30:59 -07:00
chrislu 049f040c3c refactor 2022-10-09 22:28:39 -07:00
chrislu 124b97f6d7 parsing ec volumes 2022-10-09 19:21:23 -07:00
chrislu 1e27d36576 Merge branch 'master' of https://github.com/seaweedfs/seaweedfs 2022-10-09 18:58:14 -07:00
chrislu f9383aa726 refactor to change capacity data type 2022-10-09 18:58:10 -07:00
binbinshiandshibinbin b7de4a967e fix: compact_map get error mismatching cokie (#3748)
* fix: compact_map get error

* fix: CompactSection delete lock and move test to compact_map

Co-authored-by: shibinbin <shibinbin@megvii.com>
2022-10-09 16:56:40 -07:00
BakerBunker b9d8a837af Fix mount -t weed with extra options (#3807)
fuse: Ignore "-o" command
2022-10-08 11:25:19 -07:00
Konstantin Lebedev 5db25a8f2a metric shows who is currently blocking the cluster or not (#3799)
* master_admin_lock Shows whether cluster is locked now or not
https://github.com/seaweedfs/seaweedfs/issues/3452

* fix metric MasterAdminLock
2022-10-07 13:26:29 -07:00
Viktor Kuzmin f8d3ff466d Allow parallel volume loading from different dirs during startup. (#3802) 2022-10-07 11:27:15 -07:00
Konstantin Lebedev a522507f95 configure raft metrics (#3798) 2022-10-07 04:20:34 -07:00
Konstantin Lebedev b9933d5589 master server graceful stop (#3797) 2022-10-06 09:30:30 -07:00
Konstantin Lebedev 36daa7709d show raft leader via shell (#3796) 2022-10-06 07:10:41 -07:00
dependabot[bot] 0f522da367 build(deps): bump protobuf-java from 3.16.1 to 3.16.3 in /other/java/client (#3789) 2022-10-04 17:54:30 -07:00
chrislu ea271600ec fix parameters 2022-10-04 12:36:05 -07:00
chrislu 0452ae6a6c filer.sync: limit concurrency when fetching file chunks
fix https://github.com/seaweedfs/seaweedfs/issues/3787
2022-10-04 11:35:07 -07:00
Konstantin Lebedev ec46a34f33 avoid increasing the coordinates of metric labels (#3785) 2022-10-04 01:20:01 -07:00
dependabot[bot] 18d92dbd45 build(deps): bump github.com/aws/aws-sdk-go from 1.44.105 to 1.44.110 (#3783)
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.105 to 1.44.110.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.105...v1.44.110)

---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-03 12:49:06 -07:00
dependabot[bot] 93d15488e4 build(deps): bump gocloud.dev/pubsub/rabbitpubsub from 0.26.0 to 0.27.0 (#3776)
Bumps [gocloud.dev/pubsub/rabbitpubsub](https://github.com/google/go-cloud) from 0.26.0 to 0.27.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.26.0...v0.27.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/rabbitpubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-03 12:37:08 -07:00
dependabot[bot] 595db62fe7 build(deps): bump google.golang.org/api from 0.97.0 to 0.98.0 (#3780)
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.97.0 to 0.98.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.97.0...v0.98.0)

---
updated-dependencies:
- dependency-name: google.golang.org/api
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-03 12:36:54 -07:00
dependabot[bot] 8ef802c259 build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.38.1 to 3.38.3 (#3775)
build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3

Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.38.1 to 3.38.3.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.38.1...v3.38.3)

---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-03 10:25:42 -07:00
dependabot[bot] 5b106630aa build(deps): bump gocloud.dev/pubsub/natspubsub from 0.26.0 to 0.27.0 (#3777)
Bumps [gocloud.dev/pubsub/natspubsub](https://github.com/google/go-cloud) from 0.26.0 to 0.27.0.
- [Release notes](https://github.com/google/go-cloud/releases)
- [Commits](https://github.com/google/go-cloud/compare/v0.26.0...v0.27.0)

---
updated-dependencies:
- dependency-name: gocloud.dev/pubsub/natspubsub
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-10-03 10:25:30 -07:00
dependabot[bot] 1cdbc8beb5 build(deps): bump github.com/Shopify/sarama from 1.36.0 to 1.37.0 (#3778) 2022-10-03 08:53:03 -07:00
dependabot[bot] d3f89af079 build(deps): bump github.com/google/flatbuffers from 2.0.8+incompatible to 22.9.29+incompatible (#3779) 2022-10-03 08:52:43 -07:00
chrislu 707abd5b2d 3.30 java 2022-10-02 18:54:39 -07:00
chrislu 29632d5a34 3.30 2022-10-02 18:13:44 -07:00
LHHDZ e9584d9661 add ownership rest apis (#3765) 2022-10-01 19:18:00 -07:00
Konstantin Lebedev 6fa3d0cc46 fix delete key panic of remote sync dir (#3770) 2022-10-01 02:33:47 -07:00
chrislu c3487d3da7 better fix for option format 2022-09-30 08:49:38 -07:00
chrislu ba0db28dbb mount: fix extra option format 2022-09-30 08:44:41 -07:00
chrislu 8e81619d02 mount: accept all extra mount options
fix https://github.com/seaweedfs/seaweedfs/issues/3767
2022-09-30 08:40:37 -07:00
kvtb 620be2be16 client-java: provide Content-MD5 when uploading chunks (#3761)
* client-java: provide Content-MD5 when uploading chunks

* Update SeaweedWrite.java
2022-09-29 22:56:12 -07:00
Eric Yangandroot 56c94cc08e ADHOC: filter deleted files from idx file binary search (#3763)
* ADHOC: filter deleted files from idx file binary search

* remove unwanted check

Co-authored-by: root <root@HQ-10MSTD3EY.roblox.local>
2022-09-29 12:48:36 -07:00
LHHDZ 3de1e19780 s3: sync bucket info from filer (#3759) 2022-09-29 12:29:01 -07:00
kvtb 5e9039d728 client-java: handle server HTTP errors (#3762)
* client-java: handle server HTTP errors

* Update SeaweedWrite.java
2022-09-29 12:20:14 -07:00
LHHDZ e00a12b099 associate Account and Identity by accountId (#3754) 2022-09-28 13:25:59 -07:00
LHHDZ aacdcc4cad s3: add account (#3753)
associate `Account` and `Identity` by accountId
2022-09-28 12:45:18 -07:00
Konstantin Lebedev 301b678147 [volume] Add new volumes to HUP(reload) signal (#3755)
Add new volumes to HUP(reload) signal
2022-09-28 12:44:13 -07:00
chrislu 47fd5d3fa1 skip debug message 2022-09-27 13:12:31 -07:00
Konstantin LebedevandChris Lu 7ae51d1ec6 fix set master options for single server (#3708)
Co-authored-by: Chris Lu <chrislusf@users.noreply.github.com>
2022-09-27 12:06:35 -07:00
jerebear12 190afcc3fd Allow CORS on Filer (#3750)
Adjusted filer response headers
2022-09-27 11:47:52 -07:00
Konstantin Lebedev faa6167b6b fs.meta.load load any dirs with prefix "important" (#3747)
* fs.meta.load load any dirs with prefix "important"

* replace dirPattern to dirPrefix

* help dirPrefix
2022-09-27 09:58:46 -07:00
chrislu b6d7556dda skip truncation on error
fix https://github.com/seaweedfs/seaweedfs/issues/3746
2022-09-27 09:48:23 -07:00
dependabot[bot] ea15476b06 build(deps): bump github.com/fclairamb/ftpserverlib from 0.19.1 to 0.20.0 (#3742)
build(deps): bump github.com/fclairamb/ftpserverlib

Bumps [github.com/fclairamb/ftpserverlib](https://github.com/fclairamb/ftpserverlib) from 0.19.1 to 0.20.0.
- [Release notes](https://github.com/fclairamb/ftpserverlib/releases)
- [Commits](https://github.com/fclairamb/ftpserverlib/compare/v0.19.1...v0.20.0)

---
updated-dependencies:
- dependency-name: github.com/fclairamb/ftpserverlib
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2022-09-26 09:04:12 -07:00
dependabot[bot] 53c7161f27 build(deps): bump github.com/aws/aws-sdk-go from 1.44.100 to 1.44.105 (#3744) 2022-09-26 08:43:27 -07:00
dependabot[bot] 30200cb6af build(deps): bump google.golang.org/api from 0.96.0 to 0.97.0 (#3743) 2022-09-26 08:42:06 -07:00
dependabot[bot] b5d99110d1 build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.38.0 to 3.38.1 (#3741) 2022-09-26 08:41:19 -07:00
dependabot[bot] 6b93148f9c build(deps): bump cloud.google.com/go/storage from 1.26.0 to 1.27.0 (#3740) 2022-09-26 08:40:47 -07:00
dependabot[bot] d1357c1a3a build(deps): bump actions/dependency-review-action from 2.1.0 to 2.4.0 (#3739) 2022-09-26 08:35:23 -07:00
140 changed files with 113294 additions and 1902 deletions

No files matched your search

+2 -2
View File
@@ -38,7 +38,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -87,7 +87,7 @@ jobs:
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
+1 -1
View File
@@ -28,7 +28,7 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job
steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
- uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@156f172aa3d27f3879a47cad8810b366c299e9e9 # v1.22
with:
+1 -1
View File
@@ -18,7 +18,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+1 -1
View File
@@ -16,7 +16,7 @@ jobs:
steps:
-
name: Checkout
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
-
name: Docker meta
id: docker_meta
+2 -2
View File
@@ -9,6 +9,6 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: 'Checkout Repository'
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
- name: 'Dependency Review'
uses: actions/dependency-review-action@23d1ffffb6fa5401173051ec21eba8c35242733f
uses: actions/dependency-review-action@375c5370086bfff256c37f8beec0f437e2e72ae1
+1 -1
View File
@@ -30,7 +30,7 @@ jobs:
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Install dependencies
run: |
+1 -1
View File
@@ -27,7 +27,7 @@ jobs:
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # v2
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
- name: Get dependencies
run: |
+17 -17
View File
@@ -39,63 +39,63 @@ s3tests_build:
docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
dev: build
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_race: binary_race
docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
docker compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_tls: build certstrap
ENV_FILE="tls.env" docker-compose -f compose/local-dev-compose.yml -p seaweedfs up
ENV_FILE="tls.env" docker compose -f compose/local-dev-compose.yml -p seaweedfs up
dev_mount: build
docker-compose -f compose/local-mount-compose.yml -p seaweedfs up
docker compose -f compose/local-mount-compose.yml -p seaweedfs up
run_image: build
docker run --rm -ti --device /dev/fuse --cap-add SYS_ADMIN --entrypoint /bin/sh chrislusf/seaweedfs:local
profile_mount: build
docker-compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
docker compose -f compose/local-mount-profile-compose.yml -p seaweedfs up
k8s: build
docker-compose -f compose/local-k8s-compose.yml -p seaweedfs up
docker compose -f compose/local-k8s-compose.yml -p seaweedfs up
dev_registry: build
docker-compose -f compose/local-registry-compose.yml -p seaweedfs up
docker compose -f compose/local-registry-compose.yml -p seaweedfs up
dev_replicate: build
docker-compose -f compose/local-replicate-compose.yml -p seaweedfs up
docker compose -f compose/local-replicate-compose.yml -p seaweedfs up
dev_auditlog: build
docker-compose -f compose/local-auditlog-compose.yml -p seaweedfs up
docker compose -f compose/local-auditlog-compose.yml -p seaweedfs up
dev_nextcloud: build
docker-compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
docker compose -f compose/local-nextcloud-compose.yml -p seaweedfs up
cluster: build
docker-compose -f compose/local-cluster-compose.yml -p seaweedfs up
docker compose -f compose/local-cluster-compose.yml -p seaweedfs up
2clusters: build
docker-compose -f compose/local-clusters-compose.yml -p seaweedfs up
docker compose -f compose/local-clusters-compose.yml -p seaweedfs up
2mount: build
docker-compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
docker compose -f compose/local-sync-mount-compose.yml -p seaweedfs up
hashicorp_raft: build
docker-compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
docker compose -f compose/local-hashicorp-raft-compose.yml -p seaweedfs up
s3tests: build s3tests_build
docker-compose -f compose/local-s3tests-compose.yml -p seaweedfs up
docker compose -f compose/local-s3tests-compose.yml -p seaweedfs up
filer_etcd: build
docker stack deploy -c compose/swarm-etcd.yml fs
test_etcd: build
docker-compose -f compose/test-etcd-filer.yml -p seaweedfs up
docker compose -f compose/test-etcd-filer.yml -p seaweedfs up
test_ydb: tags = ydb
test_ydb: build
export
docker-compose -f compose/test-ydb-filer.yml -p seaweedfs up
docker compose -f compose/test-ydb-filer.yml -p seaweedfs up
clean:
rm ./weed
+12 -2
View File
@@ -1,5 +1,15 @@
# Docker
## Compose V2
SeaweedFS now uses the `v2` syntax `docker compose`
If you rely on using Docker Compose as docker-compose (with a hyphen), you can set up Compose V2 to act as a drop-in replacement of the previous docker-compose. Refer to the [Installing Compose](https://docs.docker.com/compose/install/) section for detailed instructions on upgrading.
Confirm your system has docker compose v2 with a version check
```bash
$ docker compose version
Docker Compose version v2.10.2
```
## Try it out
@@ -7,7 +17,7 @@
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-compose.yml
docker-compose -f seaweedfs-compose.yml -p seaweedfs up
docker compose -f seaweedfs-compose.yml -p seaweedfs up
```
@@ -17,7 +27,7 @@ docker-compose -f seaweedfs-compose.yml -p seaweedfs up
wget https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/docker/seaweedfs-dev-compose.yml
docker-compose -f seaweedfs-dev-compose.yml -p seaweedfs up
docker compose -f seaweedfs-dev-compose.yml -p seaweedfs up
```
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
s3:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master0:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
server1:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master0:
@@ -6,7 +6,7 @@ services:
ports:
- 9333:9333
- 19333:19333
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master0 -port=9333 -peers=master1:9334,master2:9335 -mdir=/data"
volumes:
- ./master/0:/data
environment:
@@ -18,7 +18,7 @@ services:
ports:
- 9334:9334
- 19334:19334
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master1 -port=9334 -peers=master0:9333,master2:9335 -mdir=/data"
volumes:
- ./master/1:/data
environment:
@@ -30,7 +30,7 @@ services:
ports:
- 9335:9335
- 19335:19335
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
command: "-v=4 master -volumeSizeLimitMB 100 -raftHashicorp -electionTimeout 1s -ip=master2 -port=9335 -peers=master0:9333,master1:9334 -mdir=/data"
volumes:
- ./master/2:/data
environment:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
etcd:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
ydb:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+1 -1
View File
@@ -1,4 +1,4 @@
version: '2'
version: '3.9'
services:
master:
+49 -49
View File
@@ -5,16 +5,16 @@ go 1.19
require (
cloud.google.com/go v0.104.0 // indirect
cloud.google.com/go/pubsub v1.25.1
cloud.google.com/go/storage v1.26.0
cloud.google.com/go/storage v1.27.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.36.0
github.com/aws/aws-sdk-go v1.44.100
github.com/Shopify/sarama v1.37.2
github.com/aws/aws-sdk-go v1.44.114
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/disintegration/imaging v1.6.2
@@ -27,7 +27,7 @@ require (
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.19.1
github.com/fclairamb/ftpserverlib v0.20.0
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.11.5
@@ -43,10 +43,10 @@ require (
github.com/google/go-cmp v0.5.8 // indirect
github.com/google/uuid v1.3.0
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.4.0 // indirect
github.com/googleapis/gax-go/v2 v2.5.1 // indirect
github.com/gorilla/mux v1.8.0
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-uuid v1.0.3 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@@ -57,8 +57,8 @@ require (
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.15.9 // indirect
github.com/klauspost/reedsolomon v1.11.0
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/reedsolomon v1.11.1
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.7
github.com/linxGnu/grocksdb v1.7.7
@@ -66,7 +66,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -85,7 +85,7 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/spf13/afero v1.9.2 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -107,24 +107,24 @@ require (
github.com/xdg-go/stringprep v1.0.3 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.5
go.mongodb.org/mongo-driver v1.10.2
go.mongodb.org/mongo-driver v1.10.3
go.opencensus.io v0.23.0 // indirect
gocloud.dev v0.26.0
gocloud.dev/pubsub/natspubsub v0.26.0
gocloud.dev/pubsub/rabbitpubsub v0.26.0
gocloud.dev v0.27.0
gocloud.dev/pubsub/natspubsub v0.27.0
gocloud.dev/pubsub/rabbitpubsub v0.27.0
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.0.0-20220909164309-bea034e7d591
golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 // indirect
golang.org/x/sys v0.0.0-20220818161305-2296e01440c6
golang.org/x/net v0.0.0-20220927171203-f486391704dc
golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1 // indirect
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.12
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/api v0.96.0
google.golang.org/api v0.98.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20220822174746-9e6da59bd2fc // indirect
google.golang.org/grpc v1.49.0
google.golang.org/genproto v0.0.0-20220920201722-2b89144ce006 // indirect
google.golang.org/grpc v1.50.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
@@ -141,17 +141,18 @@ require (
require (
github.com/Jille/raft-grpc-transport v1.2.0
github.com/arangodb/go-driver v1.3.3
github.com/arangodb/go-driver v1.4.0
github.com/armon/go-metrics v0.4.1
github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers v2.0.8+incompatible
github.com/google/flatbuffers v22.9.29+incompatible
github.com/hanwen/go-fuse/v2 v2.1.1-0.20220627082937-d01fda7edf17
github.com/hashicorp/raft v1.3.10
github.com/hashicorp/raft v1.3.11
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/rabbitmq/amqp091-go v1.5.0
github.com/tikv/client-go/v2 v2.0.1
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.0
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
github.com/ydb-platform/ydb-go-sdk/v3 v3.38.3
golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
@@ -159,31 +160,30 @@ require (
cloud.google.com/go/compute v1.7.0 // indirect
cloud.google.com/go/iam v0.3.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.2 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.3 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.10 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.3 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.3 // indirect
github.com/aws/smithy-go v1.11.2 // indirect
github.com/aws/aws-sdk-go-v2 v1.16.8 // indirect
github.com/aws/aws-sdk-go-v2/config v1.15.15 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.12.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.15 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.9 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.16 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.9 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.17.10 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.13 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.16.10 // indirect
github.com/aws/smithy-go v1.12.0 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fclairamb/go-log v0.3.0 // indirect
github.com/fclairamb/go-log v0.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.1.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
@@ -194,17 +194,17 @@ require (
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirect
github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d // indirect
github.com/montanaflynn/stats v0.6.6 // indirect
github.com/nats-io/nats.go v1.16.0 // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/philhofer/fwd v1.1.1 // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pierrec/lz4/v4 v4.1.17 // indirect
github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
github.com/pingcap/kvproto v0.0.0-20220106070556-3fa8fa04f898 // indirect
@@ -216,7 +216,7 @@ require (
github.com/tinylib/msgp v1.1.6 // indirect
github.com/twmb/murmur3 v1.1.3 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220916101045-1c4d1accabb6 // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
@@ -224,7 +224,7 @@ require (
go.etcd.io/etcd/client/pkg/v3 v3.5.5 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
go.uber.org/zap v1.22.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
+1199 -156
View File
File diff suppressed because it is too large. Load diff
+2 -2
View File
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
appVersion: "3.29"
version: "3.29"
appVersion: "3.32"
version: "3.32"
+3 -3
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.13</version>
<version>3.30</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -14,7 +14,7 @@
</parent>
<properties>
<protobuf.version>3.16.1</protobuf.version>
<protobuf.version>3.16.3</protobuf.version>
<!-- follow https://github.com/grpc/grpc-java -->
<grpc.version>1.23.0</grpc.version>
<guava.version>30.0-jre</guava.version>
@@ -159,7 +159,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+2 -2
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.13</version>
<version>3.30</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -154,7 +154,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+2 -2
View File
@@ -5,7 +5,7 @@
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.13</version>
<version>3.30</version>
<parent>
<groupId>org.sonatype.oss</groupId>
@@ -128,7 +128,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -119,9 +119,8 @@ public class SeaweedInputStream extends InputStream {
long bytesRead = 0;
int len = buf.remaining();
int start = (int) this.position;
if (start + len <= entry.getContent().size()) {
entry.getContent().substring(start, start + len).copyTo(buf);
if (this.position< Integer.MAX_VALUE && (this.position + len )<= entry.getContent().size()) {
entry.getContent().substring((int)this.position, (int)(this.position + len)).copyTo(buf);
} else {
bytesRead = SeaweedRead.read(this.filerClient, this.visibleIntervalList, this.position, buf, SeaweedRead.fileSize(entry));
}
@@ -14,7 +14,9 @@ import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.security.SecureRandom;
import java.security.MessageDigest;
import java.util.List;
import java.util.Base64;
public class SeaweedWrite {
@@ -123,13 +125,20 @@ public class SeaweedWrite {
final byte[] bytes,
final long bytesOffset, final long bytesLength,
byte[] cipherKey) throws IOException {
MessageDigest md = null;
try {
md = MessageDigest.getInstance("MD5");
} catch (java.security.NoSuchAlgorithmException e) {
}
InputStream inputStream = null;
if (cipherKey == null || cipherKey.length == 0) {
md.update(bytes, (int) bytesOffset, (int) bytesLength);
inputStream = new ByteArrayInputStream(bytes, (int) bytesOffset, (int) bytesLength);
} else {
try {
byte[] encryptedBytes = SeaweedCipher.encrypt(bytes, (int) bytesOffset, (int) bytesLength, cipherKey);
md.update(encryptedBytes);
inputStream = new ByteArrayInputStream(encryptedBytes, 0, encryptedBytes.length);
} catch (Exception e) {
throw new IOException("fail to encrypt data", e);
@@ -140,6 +149,7 @@ public class SeaweedWrite {
if (auth != null && auth.length() != 0) {
post.addHeader("Authorization", "BEARER " + auth);
}
post.addHeader("Content-MD5", Base64.getEncoder().encodeToString(md.digest()));
post.setEntity(MultipartEntityBuilder.create()
.setMode(HttpMultipartMode.BROWSER_COMPATIBLE)
@@ -149,6 +159,13 @@ public class SeaweedWrite {
CloseableHttpResponse response = SeaweedUtil.getClosableHttpClient().execute(post);
try {
if (response.getStatusLine().getStatusCode() / 100 != 2) {
if (response.getEntity().getContentType() != null && response.getEntity().getContentType().getValue().equals("application/json")) {
throw new IOException(EntityUtils.toString(response.getEntity(), "UTF-8"));
} else {
throw new IOException(response.getStatusLine().getReasonPhrase());
}
}
String etag = response.getLastHeader("ETag").getValue();
+2 -2
View File
@@ -11,13 +11,13 @@
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-client</artifactId>
<version>3.13</version>
<version>3.30</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>com.github.chrislusf</groupId>
<artifactId>seaweedfs-hadoop2-client</artifactId>
<version>3.13</version>
<version>3.30</version>
<scope>compile</scope>
</dependency>
<dependency>
+52 -44
View File
@@ -108,7 +108,7 @@
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -124,7 +124,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.10.1</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -132,11 +132,11 @@
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-mapreduce-client-app</artifactId>
<artifactId>hadoop-yarn-api</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-yarn-api</artifactId>
<artifactId>hadoop-yarn-client</artifactId>
<groupId>org.apache.hadoop</groupId>
</exclusion>
<exclusion>
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>2.10.1</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -167,10 +167,6 @@
<artifactId>commons-math3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>xmlenc</artifactId>
<groupId>xmlenc</groupId>
</exclusion>
<exclusion>
<artifactId>commons-io</artifactId>
<groupId>commons-io</groupId>
@@ -184,20 +180,28 @@
<groupId>commons-collections</groupId>
</exclusion>
<exclusion>
<artifactId>servlet-api</artifactId>
<artifactId>javax.servlet-api</artifactId>
<groupId>javax.servlet</groupId>
</exclusion>
<exclusion>
<artifactId>jetty</artifactId>
<groupId>org.mortbay.jetty</groupId>
<artifactId>javax.activation-api</artifactId>
<groupId>javax.activation</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-server</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-util</artifactId>
<groupId>org.mortbay.jetty</groupId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-sslengine</artifactId>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jetty-webapp</artifactId>
<groupId>org.eclipse.jetty</groupId>
</exclusion>
<exclusion>
<artifactId>jsp-api</artifactId>
@@ -207,6 +211,10 @@
<artifactId>jersey-core</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-servlet</artifactId>
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>jersey-json</artifactId>
<groupId>com.sun.jersey</groupId>
@@ -216,49 +224,37 @@
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
</exclusion>
<exclusion>
<artifactId>jets3t</artifactId>
<groupId>net.java.dev.jets3t</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang</artifactId>
<groupId>commons-lang</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration</artifactId>
<groupId>commons-configuration</groupId>
</exclusion>
<exclusion>
<artifactId>commons-digester</artifactId>
<groupId>commons-digester</groupId>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
<groupId>commons-beanutils</groupId>
</exclusion>
<exclusion>
<artifactId>commons-configuration2</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>commons-lang3</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<artifactId>commons-text</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-core-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-mapper-asl</artifactId>
<groupId>org.codehaus.jackson</groupId>
</exclusion>
<exclusion>
<artifactId>avro</artifactId>
<groupId>org.apache.avro</groupId>
</exclusion>
<exclusion>
<artifactId>re2j</artifactId>
<groupId>com.google.re2j</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-auth</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -287,6 +283,14 @@
<artifactId>commons-compress</artifactId>
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>kerb-simplekdc</artifactId>
<groupId>org.apache.kerby</groupId>
</exclusion>
<exclusion>
<artifactId>jackson-databind</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
<exclusion>
<artifactId>stax2-api</artifactId>
<groupId>org.codehaus.woodstox</groupId>
@@ -295,6 +299,10 @@
<artifactId>woodstox-core</artifactId>
<groupId>com.fasterxml.woodstox</groupId>
</exclusion>
<exclusion>
<artifactId>dnsjava</artifactId>
<groupId>dnsjava</groupId>
</exclusion>
<exclusion>
<artifactId>hadoop-annotations</artifactId>
<groupId>org.apache.hadoop</groupId>
@@ -309,7 +317,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>2.10.1</hadoop.version>
<seaweedfs.client.version>3.30</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
</project>
+2 -2
View File
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<seaweedfs.client.version>3.30</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
@@ -129,7 +129,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+8 -8
View File
@@ -108,7 +108,7 @@
</plugin>
<plugin>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
@@ -124,7 +124,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>3.2.3</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>3.2.3</version>
<version>3.2.4</version>
<scope>provided</scope>
<exclusions>
<exclusion>
@@ -224,8 +224,8 @@
<groupId>com.sun.jersey</groupId>
</exclusion>
<exclusion>
<artifactId>log4j</artifactId>
<groupId>log4j</groupId>
<artifactId>reload4j</artifactId>
<groupId>ch.qos.reload4j</groupId>
</exclusion>
<exclusion>
<artifactId>commons-beanutils</artifactId>
@@ -244,7 +244,7 @@
<groupId>org.apache.commons</groupId>
</exclusion>
<exclusion>
<artifactId>slf4j-log4j12</artifactId>
<artifactId>slf4j-reload4j</artifactId>
<groupId>org.slf4j</groupId>
</exclusion>
<exclusion>
@@ -317,7 +317,7 @@
</snapshotRepository>
</distributionManagement>
<properties>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<hadoop.version>3.2.3</hadoop.version>
<seaweedfs.client.version>3.30</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
</project>
+2 -2
View File
@@ -5,7 +5,7 @@
<modelVersion>4.0.0</modelVersion>
<properties>
<seaweedfs.client.version>3.13</seaweedfs.client.version>
<seaweedfs.client.version>3.30</seaweedfs.client.version>
<hadoop.version>3.2.4</hadoop.version>
</properties>
@@ -129,7 +129,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<version>3.0.1</version>
<executions>
<execution>
<id>attach-javadocs</id>
+9 -4
View File
@@ -37,11 +37,16 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
var lastLogTsNs = time.Now().UnixNano()
processEventFnWithOffset := pb.AddOffsetFunc(func(resp *filer_pb.SubscribeMetadataResponse) error {
if *option.storageClass == "" {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
} else {
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
if resp.EventNotification.NewEntry != nil {
if *option.storageClass == "" {
if _, ok := resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass]; ok {
delete(resp.EventNotification.NewEntry.Extended, s3_constants.AmzStorageClass)
}
} else {
resp.EventNotification.NewEntry.Extended[s3_constants.AmzStorageClass] = []byte(*option.storageClass)
}
}
processor.AddSyncJob(resp)
return nil
}, 3*time.Second, func(counter int64, lastTsNs int64) error {
+10 -1
View File
@@ -47,7 +47,10 @@ func runFuse(cmd *Command, args []string) bool {
for i++; i < rawArgsLen && rawArgs[i] != ' '; i++ {
option.WriteByte(rawArgs[i])
}
options = append(options, parameter{option.String(), "true"})
// ignore "-o"
if option.String() != "o" {
options = append(options, parameter{option.String(), "true"})
}
option.Reset()
// equal separator start option with pending value
@@ -190,6 +193,12 @@ func runFuse(cmd *Command, args []string) bool {
}
case "fusermount.path":
fusermountPath = parameter.value
default:
t := parameter.name
if parameter.value != "true" {
t = fmt.Sprintf("%s=%s", parameter.name, parameter.value)
}
mountOptions.extraOptions = append(mountOptions.extraOptions, t)
}
}
+8
View File
@@ -2,6 +2,7 @@ package command
import (
"fmt"
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
@@ -253,6 +254,13 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go httpS.Serve(masterListener)
}
grace.OnInterrupt(ms.Shutdown)
grace.OnInterrupt(grpcS.Stop)
grace.OnReload(func() {
if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
ms.Topo.HashicorpRaft.LeadershipTransfer()
}
})
select {}
}
+1
View File
@@ -31,6 +31,7 @@ type MountOptions struct {
debugPort *int
localSocket *string
disableXAttr *bool
extraOptions []string
}
var (
+1 -1
View File
@@ -166,7 +166,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
// mount fuse
fuseMountOptions := &fuse.MountOptions{
AllowOther: *option.allowOthers,
Options: nil,
Options: option.extraOptions,
MaxBackground: 128,
MaxWrite: 1024 * 1024 * 2,
MaxReadAhead: 1024 * 1024 * 2,
+6 -5
View File
@@ -92,10 +92,11 @@ func init() {
masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
masterOptions.volumePreallocate = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
masterOptions.defaultReplication = cmdServer.Flag.String("master.defaultReplication", "", "Default replication type if not specified.")
masterOptions.garbageThreshold = cmdServer.Flag.Float64("garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterOptions.metricsAddress = cmdServer.Flag.String("metrics.address", "", "Prometheus gateway address")
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("resumeState", false, "resume previous state on start master server")
masterOptions.garbageThreshold = cmdServer.Flag.Float64("master.garbageThreshold", 0.3, "threshold to vacuum and reclaim spaces")
masterOptions.metricsAddress = cmdServer.Flag.String("master.metrics.address", "", "Prometheus gateway address")
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
@@ -131,7 +132,7 @@ func init() {
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
serverOptions.v.hasSlowRead = cmdServer.Flag.Bool("volume.hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
serverOptions.v.readBufferSizeMB = cmdServer.Flag.Int("volume.readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
+3 -1
View File
@@ -98,7 +98,7 @@ func init() {
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", false, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
v.hasSlowRead = cmdVolume.Flag.Bool("hasSlowRead", true, "<experimental> if true, this prevents slow reads from blocking other requests, but large file read P99 latency will increase.")
v.readBufferSizeMB = cmdVolume.Flag.Int("readBufferSizeMB", 4, "<experimental> larger values can optimize query performance but will increase some memory usage,Use with hasSlowRead normally.")
}
@@ -265,6 +265,8 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
// starting the cluster http server
clusterHttpServer := v.startClusterHttpService(volumeMux)
grace.OnReload(volumeServer.LoadNewVolumes)
stopChan := make(chan bool)
grace.OnInterrupt(func() {
fmt.Println("volume server has been killed")
+17 -10
View File
@@ -35,6 +35,22 @@ func (store *UniversalRedisStore) RollbackTransaction(ctx context.Context) error
func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
if err = store.doInsertEntry(ctx, entry); err != nil {
return err
}
dir, name := entry.FullPath.DirAndName()
if name != "" {
_, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result()
if err != nil {
return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
}
}
return nil
}
func (store *UniversalRedisStore) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
@@ -49,21 +65,12 @@ func (store *UniversalRedisStore) InsertEntry(ctx context.Context, entry *filer.
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
}
dir, name := entry.FullPath.DirAndName()
if name != "" {
_, err = store.Client.SAdd(ctx, genDirectoryListKey(dir), name).Result()
if err != nil {
return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
}
}
return nil
}
func (store *UniversalRedisStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
return store.InsertEntry(ctx, entry)
return store.doInsertEntry(ctx, entry)
}
func (store *UniversalRedisStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+19 -12
View File
@@ -47,17 +47,8 @@ func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) erro
func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
value = util.MaybeGzipData(value)
}
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
if err = store.doInsertEntry(ctx, entry); err != nil {
return err
}
dir, name := entry.FullPath.DirAndName()
@@ -74,9 +65,25 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer
return nil
}
func (store *UniversalRedis2Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
value = util.MaybeGzipData(value)
}
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
}
return nil
}
func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
return store.InsertEntry(ctx, entry)
return store.doInsertEntry(ctx, entry)
}
func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+20 -13
View File
@@ -3,11 +3,11 @@ package redis3
import (
"context"
"fmt"
"github.com/go-redsync/redsync/v4"
"time"
"github.com/go-redis/redis/v8"
redsync "github.com/go-redsync/redsync/v4"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
@@ -35,17 +35,8 @@ func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) erro
func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
value = util.MaybeGzipData(value)
}
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
if err = store.doInsertEntry(ctx, entry); err != nil {
return err
}
dir, name := entry.FullPath.DirAndName()
@@ -59,9 +50,25 @@ func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer
return nil
}
func (store *UniversalRedis3Store) doInsertEntry(ctx context.Context, entry *filer.Entry) error {
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
if len(entry.Chunks) > filer.CountEntryChunksForGzip {
value = util.MaybeGzipData(value)
}
if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
}
return nil
}
func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
return store.InsertEntry(ctx, entry)
return store.doInsertEntry(ctx, entry)
}
func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+5 -1
View File
@@ -10,7 +10,11 @@ import (
)
func ParseS3ConfigurationFromBytes[T proto.Message](content []byte, config T) error {
if err := jsonpb.Unmarshal(content, config); err != nil {
options := &jsonpb.UnmarshalOptions{
DiscardUnknown: true,
AllowPartial: true,
}
if err := options.Unmarshal(content, config); err != nil {
return err
}
return nil
+7 -3
View File
@@ -63,15 +63,19 @@ func (store *SqliteStore) initialize(dbFile, createTable, upsertQuery string) (e
var dbErr error
store.DB, dbErr = sql.Open("sqlite", dbFile)
if dbErr != nil {
store.DB.Close()
store.DB = nil
return fmt.Errorf("can not connect to %s error:%v", dbFile, err)
if store.DB != nil {
store.DB.Close()
store.DB = nil
}
return fmt.Errorf("can not connect to %s error:%v", dbFile, dbErr)
}
if err = store.DB.Ping(); err != nil {
return fmt.Errorf("connect to %s error:%v", dbFile, err)
}
store.DB.SetMaxOpenConns(1)
if err = store.CreateTable(context.Background(), abstract_sql.DEFAULT_TABLE); err != nil {
return fmt.Errorf("init table %s: %v", abstract_sql.DEFAULT_TABLE, err)
}
+22 -18
View File
@@ -9,37 +9,41 @@ import (
)
func TraverseBfs(filerClient FilerClient, parentPath util.FullPath, fn func(parentPath util.FullPath, entry *Entry)) (err error) {
K := 5
var jobQueueWg sync.WaitGroup
queue := util.NewQueue()
jobQueueWg.Add(1)
queue.Enqueue(parentPath)
var isTerminating bool
terminates := make([]chan bool, K)
for i := 0; i < K; i++ {
go func() {
terminates[i] = make(chan bool)
go func(j int) {
for {
if isTerminating {
break
select {
case <-terminates[j]:
return
default:
t := queue.Dequeue()
if t == nil {
time.Sleep(329 * time.Millisecond)
continue
}
dir := t.(util.FullPath)
processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn)
if processErr != nil {
err = processErr
}
jobQueueWg.Done()
}
t := queue.Dequeue()
if t == nil {
time.Sleep(329 * time.Millisecond)
continue
}
dir := t.(util.FullPath)
processErr := processOneDirectory(filerClient, dir, queue, &jobQueueWg, fn)
if processErr != nil {
err = processErr
}
jobQueueWg.Done()
}
}()
}(i)
}
jobQueueWg.Wait()
isTerminating = true
for i := 0; i < K; i++ {
close(terminates[i])
}
return
}
+1
View File
@@ -22,6 +22,7 @@ message Identity {
string name = 1;
repeated Credential credentials = 2;
repeated string actions = 3;
string accountId = 4;
}
message Credential {
+30 -20
View File
@@ -75,6 +75,7 @@ type Identity struct {
Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
Credentials []*Credential `protobuf:"bytes,2,rep,name=credentials,proto3" json:"credentials,omitempty"`
Actions []string `protobuf:"bytes,3,rep,name=actions,proto3" json:"actions,omitempty"`
AccountId string `protobuf:"bytes,4,opt,name=accountId,proto3" json:"accountId,omitempty"`
}
func (x *Identity) Reset() {
@@ -130,6 +131,13 @@ func (x *Identity) GetActions() []string {
return nil
}
func (x *Identity) GetAccountId() string {
if x != nil {
return x.AccountId
}
return ""
}
type Credential struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -193,26 +201,28 @@ var file_iam_proto_rawDesc = []byte{
0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x30, 0x0a, 0x0a, 0x69, 0x64, 0x65,
0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e,
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52,
0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x6e, 0x0a, 0x08, 0x49,
0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b, 0x63,
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b,
0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65, 0x6e,
0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c,
0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03,
0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x4a, 0x0a, 0x0a, 0x43,
0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63,
0x65, 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61,
0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72,
0x65, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65,
0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65,
0x65, 0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73,
0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08,
0x49, 0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62,
0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73,
0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62,
0x2f, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x0a, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x22, 0x8c, 0x01, 0x0a, 0x08,
0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x0b,
0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28,
0x0b, 0x32, 0x12, 0x2e, 0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x72, 0x65, 0x64, 0x65,
0x6e, 0x74, 0x69, 0x61, 0x6c, 0x52, 0x0b, 0x63, 0x72, 0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61,
0x6c, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20,
0x03, 0x28, 0x09, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1c, 0x0a, 0x09,
0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52,
0x09, 0x61, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x49, 0x64, 0x22, 0x4a, 0x0a, 0x0a, 0x43, 0x72,
0x65, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x61, 0x6c, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65,
0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63,
0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65,
0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63,
0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x32, 0x21, 0x0a, 0x1f, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
0x64, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4d,
0x61, 0x6e, 0x61, 0x67, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x42, 0x4b, 0x0a, 0x10, 0x73, 0x65, 0x61,
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x08, 0x49,
0x61, 0x6d, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x2d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
0x69, 0x61, 0x6d, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
+2 -1
View File
@@ -400,7 +400,8 @@ message RaftListClusterServersResponse {
message ClusterServers {
string id = 1;
string address = 2;
string suffrage = 3; //
string suffrage = 3;
bool isLeader = 4;
}
repeated ClusterServers cluster_servers = 1;
}
+120 -110
View File
@@ -3817,7 +3817,8 @@ type RaftListClusterServersResponse_ClusterServers struct {
Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
Address string `protobuf:"bytes,2,opt,name=address,proto3" json:"address,omitempty"`
Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"` //
Suffrage string `protobuf:"bytes,3,opt,name=suffrage,proto3" json:"suffrage,omitempty"`
IsLeader bool `protobuf:"varint,4,opt,name=isLeader,proto3" json:"isLeader,omitempty"`
}
func (x *RaftListClusterServersResponse_ClusterServers) Reset() {
@@ -3873,6 +3874,13 @@ func (x *RaftListClusterServersResponse_ClusterServers) GetSuffrage() string {
return ""
}
func (x *RaftListClusterServersResponse_ClusterServers) GetIsLeader() bool {
if x != nil {
return x.IsLeader
}
return false
}
var File_master_proto protoreflect.FileDescriptor
var file_master_proto_rawDesc = []byte{
@@ -4443,7 +4451,7 @@ var file_master_proto_rawDesc = []byte{
0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x1f, 0x0a, 0x1d, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xdb, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0xf7, 0x01, 0x0a, 0x1e, 0x52, 0x61, 0x66, 0x74,
0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x61, 0x0a, 0x0f, 0x63, 0x6c,
0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20,
@@ -4451,122 +4459,124 @@ var file_master_proto_rawDesc = []byte{
0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53,
0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x43,
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x0e, 0x63,
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x56, 0x0a,
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x1a, 0x72, 0x0a,
0x0e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12,
0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12,
0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09,
0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x75, 0x66,
0x66, 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x75, 0x66,
0x66, 0x72, 0x61, 0x67, 0x65, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65,
0x64, 0x12, 0x49, 0x0a, 0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65,
0x61, 0x74, 0x12, 0x14, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48,
0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d,
0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e,
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f,
0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43,
0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70,
0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73,
0x69, 0x67, 0x6e, 0x12, 0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e,
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74,
0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65,
0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e,
0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61,
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69,
0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00,
0x12, 0x5d, 0x0a, 0x10, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65,
0x6c, 0x65, 0x74, 0x65, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x66, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x69, 0x73, 0x4c, 0x65, 0x61, 0x64, 0x65,
0x72, 0x32, 0x94, 0x0d, 0x0a, 0x07, 0x53, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x12, 0x49, 0x0a,
0x0d, 0x53, 0x65, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x12, 0x14,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x48, 0x65, 0x61, 0x72, 0x74,
0x62, 0x65, 0x61, 0x74, 0x1a, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x2e, 0x48, 0x65, 0x61, 0x72, 0x74, 0x62, 0x65, 0x61, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x30, 0x01, 0x12, 0x58, 0x0a, 0x0d, 0x4b, 0x65, 0x65, 0x70,
0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x65, 0x64, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74,
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63,
0x74, 0x65, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4b, 0x65, 0x65, 0x70, 0x43, 0x6f, 0x6e, 0x6e, 0x65,
0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01,
0x30, 0x01, 0x12, 0x51, 0x0a, 0x0c, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75,
0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x06, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x12,
0x18, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69,
0x67, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x6d, 0x61, 0x73, 0x74,
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x41, 0x73, 0x73, 0x69, 0x67, 0x6e, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73,
0x74, 0x69, 0x63, 0x73, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x53,
0x74, 0x61, 0x74, 0x69, 0x73, 0x74, 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f,
0x6e, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69, 0x73, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4c, 0x69,
0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10,
0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65,
0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c,
0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44, 0x65, 0x6c, 0x65, 0x74,
0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x44,
0x65, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12,
0x4b, 0x0a, 0x0a, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e,
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65,
0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61,
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69,
0x73, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e,
0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75,
0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f,
0x6b, 0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56,
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
0x62, 0x2e, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75,
0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f,
0x6e, 0x6c, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a,
0x16, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67,
0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e,
0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65,
0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61,
0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d,
0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64,
0x65, 0x73, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4b, 0x0a, 0x0a, 0x56,
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x1c, 0x2e, 0x6d, 0x61, 0x73, 0x74,
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x57, 0x0a, 0x0e, 0x4c, 0x6f, 0x6f, 0x6b,
0x75, 0x70, 0x45, 0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x12, 0x20, 0x2e, 0x6d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45, 0x63, 0x56,
0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x6d,
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x45,
0x63, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
0x00, 0x12, 0x51, 0x0a, 0x0c, 0x56, 0x61, 0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d,
0x65, 0x12, 0x1e, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x61,
0x63, 0x75, 0x75, 0x6d, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e,
0x73, 0x65, 0x22, 0x00, 0x12, 0x63, 0x0a, 0x12, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61,
0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x12, 0x24, 0x2e, 0x6d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c, 0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72,
0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x25, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x56, 0x6f, 0x6c,
0x75, 0x6d, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x52, 0x65, 0x61, 0x64, 0x6f, 0x6e, 0x6c, 0x79, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x47, 0x65, 0x74,
0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x47, 0x65, 0x74, 0x4d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75,
0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e,
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x47, 0x65, 0x74, 0x4d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e,
0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x4c, 0x69,
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x22,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43,
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c,
0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f, 0x64, 0x65, 0x73, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, 0x6f,
0x64, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a,
0x0f, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e,
0x12, 0x21, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61,
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c,
0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23,
0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61,
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75,
0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65,
0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50,
0x69, 0x6e, 0x67, 0x12, 0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e,
0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61,
0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69,
0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73,
0x12, 0x28, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66,
0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76,
0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43,
0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41,
0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65,
0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76,
0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74,
0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72,
0x76, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a,
0x10, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65,
0x72, 0x12, 0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x4c, 0x65, 0x61,
0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x21, 0x2e, 0x6d,
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x22, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x65, 0x61, 0x73,
0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, 0x11, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65,
0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x2e, 0x6d, 0x61, 0x73,
0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x41, 0x64,
0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x24, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6c, 0x65,
0x61, 0x73, 0x65, 0x41, 0x64, 0x6d, 0x69, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73,
0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x39, 0x0a, 0x04, 0x50, 0x69, 0x6e, 0x67, 0x12,
0x16, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67,
0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72,
0x5f, 0x70, 0x62, 0x2e, 0x50, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x00, 0x12, 0x6f, 0x0a, 0x16, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c,
0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x12, 0x28, 0x2e, 0x6d,
0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73,
0x74, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52,
0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f,
0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6c, 0x75, 0x73, 0x74,
0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
0x65, 0x22, 0x00, 0x12, 0x54, 0x0a, 0x0d, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65,
0x72, 0x76, 0x65, 0x72, 0x12, 0x1f, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x41, 0x64, 0x64, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x52, 0x61, 0x66,
0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x12, 0x22, 0x2e,
0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65,
0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x61,
0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72, 0x52, 0x65,
0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70,
0x62, 0x2e, 0x52, 0x61, 0x66, 0x74, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x53, 0x65, 0x72, 0x76,
0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30,
0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65,
0x65, 0x64, 0x66, 0x73, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77,
0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62,
0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x32, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68,
0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73,
0x2f, 0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f,
0x70, 0x62, 0x2f, 0x6d, 0x61, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (
+2 -1
View File
@@ -279,7 +279,6 @@ message CopyFileResponse {
message ReadNeedleBlobRequest {
uint32 volume_id = 1;
uint64 needle_id = 2;
int64 offset = 3; // actual offset
int32 size = 4;
}
@@ -430,6 +429,7 @@ message ReadVolumeFileStatusResponse {
uint32 compaction_revision = 7;
string collection = 8;
string disk_type = 9;
VolumeInfo volume_info = 10;
}
message DiskStatus {
@@ -466,6 +466,7 @@ message VolumeInfo {
repeated RemoteFile files = 1;
uint32 version = 2;
string replication = 3;
uint32 BytesOffset = 4;
}
// tiered storage
File diff suppressed because it is too large. Load diff
@@ -24,15 +24,16 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
var wg sync.WaitGroup
for chunkIndex, sourceChunk := range sourceChunks {
wg.Add(1)
go func(chunk *filer_pb.FileChunk, index int) {
index, source := chunkIndex, sourceChunk
fs.executor.Execute(func() {
defer wg.Done()
replicatedChunk, e := fs.replicateOneChunk(chunk, path)
replicatedChunk, e := fs.replicateOneChunk(source, path)
if e != nil {
err = e
return
}
replicatedChunks[index] = replicatedChunk
}(sourceChunk, chunkIndex)
})
}
wg.Wait()
@@ -32,6 +32,7 @@ type FilerSink struct {
address string
writeChunkByFiler bool
isIncremental bool
executor *util.LimitedConcurrentExecutor
}
func init() {
@@ -83,6 +84,7 @@ func (fs *FilerSink) DoInitialize(address, grpcAddress string, dir string,
fs.diskType = diskType
fs.grpcDialOption = grpcDialOption
fs.writeChunkByFiler = writeChunkByFiler
fs.executor = util.NewLimitedConcurrentExecutor(32)
return nil
}
+33 -1
View File
@@ -2,6 +2,7 @@ package s3api
import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"net/http"
"os"
"strings"
@@ -16,6 +17,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
)
var IdentityAnonymous *Identity
type Action string
type Iam interface {
@@ -32,10 +35,15 @@ type IdentityAccessManagement struct {
type Identity struct {
Name string
AccountId string
Credentials []*Credential
Actions []Action
}
func (i *Identity) isAnonymous() bool {
return i.Name == s3account.AccountAnonymous.Name
}
type Credential struct {
AccessKey string
SecretKey string
@@ -125,9 +133,23 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
for _, ident := range config.Identities {
t := &Identity{
Name: ident.Name,
AccountId: s3account.AccountAdmin.Id,
Credentials: nil,
Actions: nil,
}
if ident.Name == s3account.AccountAnonymous.Name {
if ident.AccountId != "" && ident.AccountId != s3account.AccountAnonymous.Id {
glog.Warningf("anonymous identity is associated with a non-anonymous account ID, the association is invalid")
}
t.AccountId = s3account.AccountAnonymous.Id
IdentityAnonymous = t
} else {
if len(ident.AccountId) > 0 {
t.AccountId = ident.AccountId
}
}
for _, action := range ident.Actions {
t.Actions = append(t.Actions, Action(action))
}
@@ -139,6 +161,13 @@ func (iam *IdentityAccessManagement) loadS3ApiConfiguration(config *iam_pb.S3Api
}
identities = append(identities, t)
}
if IdentityAnonymous == nil {
IdentityAnonymous = &Identity{
Name: s3account.AccountAnonymous.Name,
AccountId: s3account.AccountAnonymous.Id,
}
}
iam.m.Lock()
// atomically switch
iam.identities = identities
@@ -173,7 +202,7 @@ func (iam *IdentityAccessManagement) lookupAnonymous() (identity *Identity, foun
iam.m.RLock()
defer iam.m.RUnlock()
for _, ident := range iam.identities {
if ident.Name == "anonymous" {
if ident.isAnonymous() {
return ident, true
}
}
@@ -259,6 +288,9 @@ func (iam *IdentityAccessManagement) authRequest(r *http.Request, action Action)
return identity, s3err.ErrAccessDenied
}
if !identity.isAnonymous() {
r.Header.Set(s3_constants.AmzAccountId, identity.AccountId)
}
return identity, s3err.ErrNone
}
+17 -2
View File
@@ -9,7 +9,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
)
func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, lastTsNs int64) {
func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, lastTsNs int64, prefix string, directoriesToWatch []string) {
processEventFn := func(resp *filer_pb.SubscribeMetadataResponse) error {
@@ -28,6 +28,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
_ = s3a.onIamConfigUpdate(dir, fileName, content)
_ = s3a.onCircuitBreakerConfigUpdate(dir, fileName, content)
_ = s3a.onBucketMetadataChange(dir, message.OldEntry, message.NewEntry)
return nil
}
@@ -35,7 +36,7 @@ func (s3a *S3ApiServer) subscribeMetaEvents(clientName string, prefix string, la
var clientEpoch int32
util.RetryForever("followIamChanges", func() error {
clientEpoch++
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, nil, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
return pb.WithFilerClientFollowMetadata(s3a, clientName, s3a.randomClientId, clientEpoch, prefix, directoriesToWatch, &lastTsNs, 0, 0, processEventFn, pb.FatalOnError)
}, func(err error) bool {
glog.V(0).Infof("iam follow metadata changes: %v", err)
return true
@@ -63,3 +64,17 @@ func (s3a *S3ApiServer) onCircuitBreakerConfigUpdate(dir, filename string, conte
}
return nil
}
//reload bucket metadata
func (s3a *S3ApiServer) onBucketMetadataChange(dir string, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) error {
if dir == s3a.option.BucketsPath {
if newEntry != nil {
s3a.bucketRegistry.LoadBucketMetadata(newEntry)
glog.V(0).Infof("updated bucketMetadata %s/%s", dir, newEntry)
} else {
s3a.bucketRegistry.RemoveBucketMetadata(oldEntry)
glog.V(0).Infof("remove bucketMetadata %s/%s", dir, newEntry)
}
}
return nil
}
+95
View File
@@ -2,7 +2,9 @@ package s3api
import (
. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/stretchr/testify/assert"
"reflect"
"testing"
jsonpb "google.golang.org/protobuf/encoding/protojson"
@@ -124,5 +126,98 @@ func TestCanDo(t *testing.T) {
}
assert.Equal(t, true, ident5.canDo(ACTION_READ, "special_bucket", "/a/b/c/d.txt"))
assert.Equal(t, true, ident5.canDo(ACTION_WRITE, "special_bucket", "/a/b/c/d.txt"))
}
type LoadS3ApiConfigurationTestCase struct {
pbIdent *iam_pb.Identity
expectIdent *Identity
}
func TestLoadS3ApiConfiguration(t *testing.T) {
testCases := map[string]*LoadS3ApiConfigurationTestCase{
"notSpecifyAccountId": {
pbIdent: &iam_pb.Identity{
Name: "notSpecifyAccountId",
Actions: []string{
"Read",
"Write",
},
Credentials: []*iam_pb.Credential{
{
AccessKey: "some_access_key1",
SecretKey: "some_secret_key2",
},
},
},
expectIdent: &Identity{
Name: "notSpecifyAccountId",
AccountId: s3account.AccountAdmin.Id,
Actions: []Action{
"Read",
"Write",
},
Credentials: []*Credential{
{
AccessKey: "some_access_key1",
SecretKey: "some_secret_key2",
},
},
},
},
"specifiedAccountID": {
pbIdent: &iam_pb.Identity{
Name: "specifiedAccountID",
AccountId: "specifiedAccountID",
Actions: []string{
"Read",
"Write",
},
},
expectIdent: &Identity{
Name: "specifiedAccountID",
AccountId: "specifiedAccountID",
Actions: []Action{
"Read",
"Write",
},
},
},
"anonymous": {
pbIdent: &iam_pb.Identity{
Name: "anonymous",
Actions: []string{
"Read",
"Write",
},
},
expectIdent: &Identity{
Name: "anonymous",
AccountId: "anonymous",
Actions: []Action{
"Read",
"Write",
},
},
},
}
config := &iam_pb.S3ApiConfiguration{
Identities: make([]*iam_pb.Identity, 0),
}
for _, v := range testCases {
config.Identities = append(config.Identities, v.pbIdent)
}
iam := IdentityAccessManagement{}
err := iam.loadS3ApiConfiguration(config)
if err != nil {
return
}
for _, ident := range iam.identities {
tc := testCases[ident.Name]
if !reflect.DeepEqual(ident, tc.expectIdent) {
t.Error("not expect")
}
}
}
+217
View File
@@ -0,0 +1,217 @@
package s3api
import (
"encoding/json"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"sync"
)
var loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
entry, err := filer_pb.GetEntry(r.s3a, util.NewFullPath(r.s3a.option.BucketsPath, bucketName))
if err != nil {
return nil, err
}
return buildBucketMetadata(r.s3a.accountManager, entry), nil
}
type BucketMetaData struct {
_ struct{} `type:"structure"`
Name string
//By default, when another AWS account uploads an object to S3 bucket,
//that account (the object writer) owns the object, has access to it, and
//can grant other users access to it through ACLs. You can use Object Ownership
//to change this default behavior so that ACLs are disabled and you, as the
//bucket owner, automatically own every object in your bucket.
ObjectOwnership string
// Container for the bucket owner's display name and ID.
Owner *s3.Owner `type:"structure"`
// A list of grants for access controls.
Acl []*s3.Grant `locationName:"AccessControlList" locationNameList:"Grant" type:"list"`
}
type BucketRegistry struct {
metadataCache map[string]*BucketMetaData
metadataCacheLock sync.RWMutex
notFound map[string]struct{}
notFoundLock sync.RWMutex
s3a *S3ApiServer
}
func NewBucketRegistry(s3a *S3ApiServer) *BucketRegistry {
br := &BucketRegistry{
metadataCache: make(map[string]*BucketMetaData),
notFound: make(map[string]struct{}),
s3a: s3a,
}
err := br.init()
if err != nil {
glog.Fatal("init bucket registry failed", err)
return nil
}
return br
}
func (r *BucketRegistry) init() error {
err := filer_pb.List(r.s3a, r.s3a.option.BucketsPath, "", func(entry *filer_pb.Entry, isLast bool) error {
r.LoadBucketMetadata(entry)
return nil
}, "", false, math.MaxUint32)
return err
}
func (r *BucketRegistry) LoadBucketMetadata(entry *filer_pb.Entry) {
bucketMetadata := buildBucketMetadata(r.s3a.accountManager, entry)
r.metadataCacheLock.Lock()
defer r.metadataCacheLock.Unlock()
r.metadataCache[entry.Name] = bucketMetadata
}
func buildBucketMetadata(accountManager *s3account.AccountManager, entry *filer_pb.Entry) *BucketMetaData {
entryJson, _ := json.Marshal(entry)
glog.V(3).Infof("build bucket metadata,entry=%s", entryJson)
bucketMetadata := &BucketMetaData{
Name: entry.Name,
//Default ownership: OwnershipBucketOwnerEnforced, which means Acl is disabled
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
// Default owner: `AccountAdmin`
Owner: &s3.Owner{
ID: &s3account.AccountAdmin.Id,
DisplayName: &s3account.AccountAdmin.Name,
},
}
if entry.Extended != nil {
//ownership control
ownership, ok := entry.Extended[s3_constants.ExtOwnershipKey]
if ok {
ownership := string(ownership)
valid := s3_constants.ValidateOwnership(ownership)
if valid {
bucketMetadata.ObjectOwnership = ownership
} else {
glog.Warningf("Invalid ownership: %s, bucket: %s", ownership, bucketMetadata.Name)
}
}
//access control policy
//owner
acpOwnerBytes, ok := entry.Extended[s3_constants.ExtAmzOwnerKey]
if ok && len(acpOwnerBytes) > 0 {
ownerAccountId := string(acpOwnerBytes)
ownerAccountName, exists := accountManager.IdNameMapping[ownerAccountId]
if !exists {
glog.Warningf("owner[id=%s] is invalid, bucket: %s", ownerAccountId, bucketMetadata.Name)
} else {
bucketMetadata.Owner = &s3.Owner{
ID: &ownerAccountId,
DisplayName: &ownerAccountName,
}
}
}
//grants
acpGrantsBytes, ok := entry.Extended[s3_constants.ExtAmzAclKey]
if ok && len(acpGrantsBytes) > 0 {
var grants []*s3.Grant
err := json.Unmarshal(acpGrantsBytes, &grants)
if err == nil {
bucketMetadata.Acl = grants
} else {
glog.Warningf("Unmarshal ACP grants: %s(%v), bucket: %s", string(acpGrantsBytes), err, bucketMetadata.Name)
}
}
}
return bucketMetadata
}
func (r *BucketRegistry) RemoveBucketMetadata(entry *filer_pb.Entry) {
r.removeMetadataCache(entry.Name)
r.unMarkNotFound(entry.Name)
}
func (r *BucketRegistry) GetBucketMetadata(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
r.metadataCacheLock.RLock()
bucketMetadata, ok := r.metadataCache[bucketName]
r.metadataCacheLock.RUnlock()
if ok {
return bucketMetadata, s3err.ErrNone
}
r.notFoundLock.RLock()
_, ok = r.notFound[bucketName]
r.notFoundLock.RUnlock()
if ok {
return nil, s3err.ErrNoSuchBucket
}
bucketMetadata, errCode := r.LoadBucketMetadataFromFiler(bucketName)
if errCode != s3err.ErrNone {
return nil, errCode
}
r.setMetadataCache(bucketMetadata)
r.unMarkNotFound(bucketName)
return bucketMetadata, s3err.ErrNone
}
func (r *BucketRegistry) LoadBucketMetadataFromFiler(bucketName string) (*BucketMetaData, s3err.ErrorCode) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()
//check if already exists
r.metadataCacheLock.RLock()
bucketMetaData, ok := r.metadataCache[bucketName]
r.metadataCacheLock.RUnlock()
if ok {
return bucketMetaData, s3err.ErrNone
}
//if not exists, load from filer
bucketMetadata, err := loadBucketMetadataFromFiler(r, bucketName)
if err != nil {
if err == filer_pb.ErrNotFound {
// The bucket doesn't actually exist and should no longer loaded from the filer
r.notFound[bucketName] = struct{}{}
return nil, s3err.ErrNoSuchBucket
}
return nil, s3err.ErrInternalError
}
return bucketMetadata, s3err.ErrNone
}
func (r *BucketRegistry) setMetadataCache(metadata *BucketMetaData) {
r.metadataCacheLock.Lock()
defer r.metadataCacheLock.Unlock()
r.metadataCache[metadata.Name] = metadata
}
func (r *BucketRegistry) removeMetadataCache(bucket string) {
r.metadataCacheLock.Lock()
defer r.metadataCacheLock.Unlock()
delete(r.metadataCache, bucket)
}
func (r *BucketRegistry) markNotFound(bucket string) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()
r.notFound[bucket] = struct{}{}
}
func (r *BucketRegistry) unMarkNotFound(bucket string) {
r.notFoundLock.Lock()
defer r.notFoundLock.Unlock()
delete(r.notFound, bucket)
}
+231
View File
@@ -0,0 +1,231 @@
package s3api
import (
"encoding/json"
"fmt"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"reflect"
"sync"
"testing"
"time"
)
type BucketMetadataTestCase struct {
filerEntry *filer_pb.Entry
expectBucketMetadata *BucketMetaData
}
var (
//bad entry
badEntry = &filer_pb.Entry{
Name: "badEntry",
}
//good entry
goodEntryAcl, _ = json.Marshal(s3_constants.PublicRead)
goodEntry = &filer_pb.Entry{
Name: "entryWithValidAcp",
Extended: map[string][]byte{
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
s3_constants.ExtAmzOwnerKey: []byte(s3account.AccountAdmin.Name),
s3_constants.ExtAmzAclKey: goodEntryAcl,
},
}
//ownership is ""
ownershipEmptyStr = &filer_pb.Entry{
Name: "ownershipEmptyStr",
Extended: map[string][]byte{
s3_constants.ExtOwnershipKey: []byte(""),
},
}
//ownership valid
ownershipValid = &filer_pb.Entry{
Name: "ownershipValid",
Extended: map[string][]byte{
s3_constants.ExtOwnershipKey: []byte(s3_constants.OwnershipBucketOwnerEnforced),
},
}
//owner is ""
acpEmptyStr = &filer_pb.Entry{
Name: "acpEmptyStr",
Extended: map[string][]byte{
s3_constants.ExtAmzOwnerKey: []byte(""),
},
}
//owner not exists
acpEmptyObject = &filer_pb.Entry{
Name: "acpEmptyObject",
Extended: map[string][]byte{
s3_constants.ExtAmzOwnerKey: []byte("xxxxx"),
},
}
//grants is nil
acpOwnerNilAcp, _ = json.Marshal(make([]*s3.Grant, 0))
acpOwnerNil = &filer_pb.Entry{
Name: "acpOwnerNil",
Extended: map[string][]byte{
s3_constants.ExtAmzAclKey: acpOwnerNilAcp,
},
}
//load filer is
loadFilerBucket = make(map[string]int, 1)
//override `loadBucketMetadataFromFiler` to avoid really load from filer
)
var tcs = []*BucketMetadataTestCase{
{
badEntry, &BucketMetaData{
Name: badEntry.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: nil,
},
},
{
goodEntry, &BucketMetaData{
Name: goodEntry.Name,
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: s3_constants.PublicRead,
},
},
{
ownershipEmptyStr, &BucketMetaData{
Name: ownershipEmptyStr.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: nil,
},
},
{
ownershipValid, &BucketMetaData{
Name: ownershipValid.Name,
ObjectOwnership: s3_constants.OwnershipBucketOwnerEnforced,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: nil,
},
},
{
acpEmptyStr, &BucketMetaData{
Name: acpEmptyStr.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: nil,
},
},
{
acpEmptyObject, &BucketMetaData{
Name: acpEmptyObject.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: nil,
},
},
{
acpOwnerNil, &BucketMetaData{
Name: acpOwnerNil.Name,
ObjectOwnership: s3_constants.DefaultOwnershipForExists,
Owner: &s3.Owner{
DisplayName: &s3account.AccountAdmin.Name,
ID: &s3account.AccountAdmin.Id,
},
Acl: make([]*s3.Grant, 0),
},
},
}
func TestBuildBucketMetadata(t *testing.T) {
accountManager := &s3account.AccountManager{
IdNameMapping: map[string]string{
s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
},
}
for _, tc := range tcs {
resultBucketMetadata := buildBucketMetadata(accountManager, tc.filerEntry)
if !reflect.DeepEqual(resultBucketMetadata, tc.expectBucketMetadata) {
t.Fatalf("result is unexpect: \nresult: %v, \nexpect: %v", resultBucketMetadata, tc.expectBucketMetadata)
}
}
}
func TestGetBucketMetadata(t *testing.T) {
loadBucketMetadataFromFiler = func(r *BucketRegistry, bucketName string) (*BucketMetaData, error) {
time.Sleep(time.Second)
loadFilerBucket[bucketName] = loadFilerBucket[bucketName] + 1
return &BucketMetaData{
Name: bucketName,
}, nil
}
br := &BucketRegistry{
metadataCache: make(map[string]*BucketMetaData),
notFound: make(map[string]struct{}),
s3a: nil,
}
//start 40 goroutine for
var wg sync.WaitGroup
closeCh := make(chan struct{})
for i := 0; i < 40; i++ {
wg.Add(1)
go func() {
defer wg.Done()
outLoop:
for {
for j := 0; j < 5; j++ {
select {
case <-closeCh:
break outLoop
default:
reqBucket := fmt.Sprintf("%c", 67+j)
_, errCode := br.GetBucketMetadata(reqBucket)
if errCode != s3err.ErrNone {
close(closeCh)
t.Error("not expect")
}
}
}
time.Sleep(10 * time.Microsecond)
}
}()
}
time.Sleep(time.Second)
close(closeCh)
wg.Wait()
//Each bucket is loaded from the filer only once
for bucketName, loadCount := range loadFilerBucket {
if loadCount != 1 {
t.Fatalf("lock is uneffict: %s, %d", bucketName, loadCount)
}
}
}
+16
View File
@@ -91,6 +91,22 @@ func (s3a *S3ApiServer) getEntry(parentDirectoryPath, entryName string) (entry *
return filer_pb.GetEntry(s3a, fullPath)
}
func (s3a *S3ApiServer) updateEntry(parentDirectoryPath string, newEntry *filer_pb.Entry) error {
updateEntryRequest := &filer_pb.UpdateEntryRequest{
Directory: parentDirectoryPath,
Entry: newEntry,
}
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
err := filer_pb.UpdateEntry(client, updateEntryRequest)
if err != nil {
return err
}
return nil
})
return err
}
func objectKey(key *string) *string {
if strings.HasPrefix(*key, "/") {
t := (*key)[1:]
+65
View File
@@ -0,0 +1,65 @@
package s3_constants
import (
"github.com/aws/aws-sdk-go/service/s3"
)
const (
CannedAclPrivate = "private"
CannedAclPublicRead = "public-read"
CannedAclPublicReadWrite = "public-read-write"
CannedAclAuthenticatedRead = "authenticated-read"
CannedAclLogDeliveryWrite = "log-delivery-write"
CannedAclBucketOwnerRead = "bucket-owner-read"
CannedAclBucketOwnerFullControl = "bucket-owner-full-control"
CannedAclAwsExecRead = "aws-exec-read"
)
var (
PublicRead = []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &GrantTypeGroup,
URI: &GranteeGroupAllUsers,
},
Permission: &PermissionRead,
},
}
PublicReadWrite = []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &GrantTypeGroup,
URI: &GranteeGroupAllUsers,
},
Permission: &PermissionRead,
},
{
Grantee: &s3.Grantee{
Type: &GrantTypeGroup,
URI: &GranteeGroupAllUsers,
},
Permission: &PermissionWrite,
},
}
AuthenticatedRead = []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &GrantTypeGroup,
URI: &GranteeGroupAuthenticatedUsers,
},
Permission: &PermissionRead,
},
}
LogDeliveryWrite = []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &GrantTypeGroup,
URI: &GranteeGroupLogDelivery,
},
Permission: &PermissionWrite,
},
}
)
@@ -0,0 +1,20 @@
package s3_constants
//Amazon S3 predefined groups
var (
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
GranteeGroupLogDelivery = "http://acs.amazonaws.com/groups/s3/LogDelivery"
)
func ValidateGroup(group string) bool {
valid := true
switch group {
case GranteeGroupAllUsers:
case GranteeGroupLogDelivery:
case GranteeGroupAuthenticatedUsers:
default:
valid = false
}
return valid
}
@@ -0,0 +1,7 @@
package s3_constants
var (
GrantTypeCanonicalUser = "CanonicalUser"
GrantTypeAmazonCustomerByEmail = "AmazonCustomerByEmail"
GrantTypeGroup = "Group"
)
+18
View File
@@ -0,0 +1,18 @@
package s3_constants
var (
OwnershipBucketOwnerPreferred = "BucketOwnerPreferred"
OwnershipObjectWriter = "ObjectWriter"
OwnershipBucketOwnerEnforced = "BucketOwnerEnforced"
DefaultOwnershipForCreate = OwnershipObjectWriter
DefaultOwnershipForExists = OwnershipBucketOwnerEnforced
)
func ValidateOwnership(ownership string) bool {
if ownership == "" || (ownership != OwnershipBucketOwnerPreferred && ownership != OwnershipObjectWriter && ownership != OwnershipBucketOwnerEnforced) {
return false
} else {
return true
}
}
+9
View File
@@ -0,0 +1,9 @@
package s3_constants
var (
PermissionFullControl = "FULL_CONTROL"
PermissionRead = "READ"
PermissionWrite = "WRITE"
PermissionReadAcp = "READ_ACP"
PermissionWriteAcp = "WRITE_ACP"
)
+7
View File
@@ -0,0 +1,7 @@
package s3_constants
const (
ExtAmzOwnerKey = "Seaweed-X-Amz-Owner"
ExtAmzAclKey = "Seaweed-X-Amz-Acl"
ExtOwnershipKey = "Seaweed-X-Amz-Ownership"
)
+9
View File
@@ -38,11 +38,20 @@ const (
AmzTagCount = "x-amz-tagging-count"
X_SeaweedFS_Header_Directory_Key = "x-seaweedfs-is-directory-key"
// S3 ACL headers
AmzCannedAcl = "X-Amz-Acl"
AmzAclFullControl = "X-Amz-Grant-Full-Control"
AmzAclRead = "X-Amz-Grant-Read"
AmzAclWrite = "X-Amz-Grant-Write"
AmzAclReadAcp = "X-Amz-Grant-Read-Acp"
AmzAclWriteAcp = "X-Amz-Grant-Write-Acp"
)
// Non-Standard S3 HTTP request constants
const (
AmzIdentityId = "s3-identity-id"
AmzAccountId = "s3-account-id"
AmzAuthType = "s3-auth-type"
AmzIsAdmin = "s3-is-admin" // only set to http request header as a context
)
+70
View File
@@ -0,0 +1,70 @@
package s3account
import (
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"sync"
)
//Predefined Accounts
var (
// AccountAdmin is used as the default account for IAM-Credentials access without Account configured
AccountAdmin = Account{
Name: "admin",
EmailAddress: "admin@example.com",
Id: "admin",
}
// AccountAnonymous is used to represent the account for anonymous access
AccountAnonymous = Account{
Name: "anonymous",
EmailAddress: "anonymous@example.com",
Id: "anonymous",
}
)
//Account represents a system user, a system user can
//configure multiple IAM-Users, IAM-Users can configure
//permissions respectively, and each IAM-User can
//configure multiple security credentials
type Account struct {
//Name is also used to display the "DisplayName" as the owner of the bucket or object
Name string
EmailAddress string
//Id is used to identify an Account when granting cross-account access(ACLs) to buckets and objects
Id string
}
type AccountManager struct {
sync.Mutex
filerClient filer_pb.FilerClient
IdNameMapping map[string]string
EmailIdMapping map[string]string
}
func NewAccountManager(filerClient filer_pb.FilerClient) *AccountManager {
am := &AccountManager{
filerClient: filerClient,
IdNameMapping: make(map[string]string),
EmailIdMapping: make(map[string]string),
}
am.initialize()
return am
}
func (am *AccountManager) GetAccountNameById(canonicalId string) string {
return am.IdNameMapping[canonicalId]
}
func (am *AccountManager) GetAccountIdByEmail(email string) string {
return am.EmailIdMapping[email]
}
func (am *AccountManager) initialize() {
// load predefined Accounts
for _, account := range []Account{AccountAdmin, AccountAnonymous} {
am.IdNameMapping[account.Id] = account.Name
am.EmailIdMapping[account.EmailAddress] = account.Id
}
}
+509
View File
@@ -0,0 +1,509 @@
package s3acl
import (
"encoding/json"
"encoding/xml"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/util"
"net/http"
"strings"
)
// GetAccountId get AccountId from request headers, AccountAnonymousId will be return if not presen
func GetAccountId(r *http.Request) string {
id := r.Header.Get(s3_constants.AmzAccountId)
if len(id) == 0 {
return s3account.AccountAnonymous.Id
} else {
return id
}
}
// ExtractAcl extracts the acl from the request body, or from the header if request body is empty
func ExtractAcl(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, ownerId, accountId string) (grants []*s3.Grant, errCode s3err.ErrorCode) {
if r.Body != nil && r.Body != http.NoBody {
defer util.CloseRequest(r)
var acp s3.AccessControlPolicy
err := xmlutil.UnmarshalXML(&acp, xml.NewDecoder(r.Body), "")
if err != nil || acp.Owner == nil || acp.Owner.ID == nil {
return nil, s3err.ErrInvalidRequest
}
//owner should present && owner is immutable
if *acp.Owner.ID != ownerId {
glog.V(3).Infof("set acl denied! owner account is not consistent, request account id: %s, expect account id: %s", accountId, ownerId)
return nil, s3err.ErrAccessDenied
}
return ValidateAndTransferGrants(accountManager, acp.Grants)
} else {
_, grants, errCode = ParseAndValidateAclHeadersOrElseDefault(r, accountManager, ownership, bucketOwnerId, accountId, true)
return grants, errCode
}
}
// ParseAndValidateAclHeadersOrElseDefault will callParseAndValidateAclHeaders to get Grants, if empty, it will return Grant that grant `accountId` with `FullControl` permission
func ParseAndValidateAclHeadersOrElseDefault(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
ownerId, grants, errCode = ParseAndValidateAclHeaders(r, accountManager, ownership, bucketOwnerId, accountId, putAcl)
if errCode != s3err.ErrNone {
return
}
if len(grants) == 0 {
//if no acl(both customAcl and cannedAcl) specified, grant accountId(object writer) with full control permission
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
})
}
return
}
// ParseAndValidateAclHeaders parse and validate acl from header
func ParseAndValidateAclHeaders(r *http.Request, accountManager *s3account.AccountManager, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
ownerId, grants, errCode = ParseAclHeaders(r, ownership, bucketOwnerId, accountId, putAcl)
if errCode != s3err.ErrNone {
return
}
if len(grants) > 0 {
grants, errCode = ValidateAndTransferGrants(accountManager, grants)
}
return
}
// ParseAclHeaders parse acl headers
// When `putAcl` is true, only `CannedAcl` is parsed, such as `PutBucketAcl` or `PutObjectAcl`
// is requested, `CustomAcl` is parsed from the request body not from headers, and only if the
// request body is empty, `CannedAcl` is parsed from the header, and will not parse `CustomAcl` from the header
//
// Since `CustomAcl` has higher priority, it will be parsed first; if `CustomAcl` does not exist, `CannedAcl` will be parsed
func ParseAclHeaders(r *http.Request, ownership, bucketOwnerId, accountId string, putAcl bool) (ownerId string, grants []*s3.Grant, errCode s3err.ErrorCode) {
if !putAcl {
errCode = ParseCustomAclHeaders(r, &grants)
if errCode != s3err.ErrNone {
return "", nil, errCode
}
}
if len(grants) > 0 {
return accountId, grants, s3err.ErrNone
}
cannedAcl := r.Header.Get(s3_constants.AmzCannedAcl)
if len(cannedAcl) == 0 {
return accountId, grants, s3err.ErrNone
}
//if canned acl specified, parse cannedAcl (lower priority to custom acl)
ownerId, grants, errCode = ParseCannedAclHeader(ownership, bucketOwnerId, accountId, cannedAcl, putAcl)
if errCode != s3err.ErrNone {
return "", nil, errCode
}
return ownerId, grants, errCode
}
func ParseCustomAclHeaders(r *http.Request, grants *[]*s3.Grant) s3err.ErrorCode {
customAclHeaders := []string{s3_constants.AmzAclFullControl, s3_constants.AmzAclRead, s3_constants.AmzAclReadAcp, s3_constants.AmzAclWrite, s3_constants.AmzAclWriteAcp}
var errCode s3err.ErrorCode
for _, customAclHeader := range customAclHeaders {
headerValue := r.Header.Get(customAclHeader)
switch customAclHeader {
case s3_constants.AmzAclRead:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionRead, grants)
case s3_constants.AmzAclWrite:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWrite, grants)
case s3_constants.AmzAclReadAcp:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionReadAcp, grants)
case s3_constants.AmzAclWriteAcp:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionWriteAcp, grants)
case s3_constants.AmzAclFullControl:
errCode = ParseCustomAclHeader(headerValue, s3_constants.PermissionFullControl, grants)
}
if errCode != s3err.ErrNone {
return errCode
}
}
return s3err.ErrNone
}
func ParseCustomAclHeader(headerValue, permission string, grants *[]*s3.Grant) s3err.ErrorCode {
if len(headerValue) > 0 {
split := strings.Split(headerValue, ", ")
for _, grantStr := range split {
kv := strings.Split(grantStr, "=")
if len(kv) != 2 {
return s3err.ErrInvalidRequest
}
switch kv[0] {
case "id":
var accountId string
_ = json.Unmarshal([]byte(kv[1]), &accountId)
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &permission,
})
case "emailAddress":
var emailAddress string
_ = json.Unmarshal([]byte(kv[1]), &emailAddress)
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeAmazonCustomerByEmail,
EmailAddress: &emailAddress,
},
Permission: &permission,
})
case "uri":
var groupName string
_ = json.Unmarshal([]byte(kv[1]), &groupName)
*grants = append(*grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &groupName,
},
Permission: &permission,
})
}
}
}
return s3err.ErrNone
}
func ParseCannedAclHeader(bucketOwnership, bucketOwnerId, accountId, cannedAcl string, putAcl bool) (ownerId string, grants []*s3.Grant, err s3err.ErrorCode) {
err = s3err.ErrNone
ownerId = accountId
//objectWrite automatically has full control on current object
objectWriterFullControl := &s3.Grant{
Grantee: &s3.Grantee{
ID: &accountId,
Type: &s3_constants.GrantTypeCanonicalUser,
},
Permission: &s3_constants.PermissionFullControl,
}
switch cannedAcl {
case s3_constants.CannedAclPrivate:
grants = append(grants, objectWriterFullControl)
case s3_constants.CannedAclPublicRead:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.PublicRead...)
case s3_constants.CannedAclPublicReadWrite:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.PublicReadWrite...)
case s3_constants.CannedAclAuthenticatedRead:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.AuthenticatedRead...)
case s3_constants.CannedAclLogDeliveryWrite:
grants = append(grants, objectWriterFullControl)
grants = append(grants, s3_constants.LogDeliveryWrite...)
case s3_constants.CannedAclBucketOwnerRead:
grants = append(grants, objectWriterFullControl)
if bucketOwnerId != "" && bucketOwnerId != accountId {
grants = append(grants,
&s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwnerId,
},
Permission: &s3_constants.PermissionRead,
})
}
case s3_constants.CannedAclBucketOwnerFullControl:
if bucketOwnerId != "" {
// if set ownership to 'BucketOwnerPreferred' when upload object, the bucket owner will be the object owner
if !putAcl && bucketOwnership == s3_constants.OwnershipBucketOwnerPreferred {
ownerId = bucketOwnerId
grants = append(grants,
&s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwnerId,
},
Permission: &s3_constants.PermissionFullControl,
})
} else {
grants = append(grants, objectWriterFullControl)
if accountId != bucketOwnerId {
grants = append(grants,
&s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwnerId,
},
Permission: &s3_constants.PermissionFullControl,
})
}
}
}
case s3_constants.CannedAclAwsExecRead:
err = s3err.ErrNotImplemented
default:
err = s3err.ErrInvalidRequest
}
return
}
// ValidateAndTransferGrants validate grant & transfer Email-Grant to Id-Grant
func ValidateAndTransferGrants(accountManager *s3account.AccountManager, grants []*s3.Grant) ([]*s3.Grant, s3err.ErrorCode) {
var result []*s3.Grant
for _, grant := range grants {
grantee := grant.Grantee
if grantee == nil || grantee.Type == nil {
glog.Warning("invalid grantee! grantee or granteeType is nil")
return nil, s3err.ErrInvalidRequest
}
switch *grantee.Type {
case s3_constants.GrantTypeGroup:
if grantee.URI == nil {
glog.Warning("invalid group grantee! group URI is nil")
return nil, s3err.ErrInvalidRequest
}
ok := s3_constants.ValidateGroup(*grantee.URI)
if !ok {
glog.Warningf("invalid group grantee! group name[%s] is not valid", *grantee.URI)
return nil, s3err.ErrInvalidRequest
}
result = append(result, grant)
case s3_constants.GrantTypeCanonicalUser:
if grantee.ID == nil {
glog.Warning("invalid canonical grantee! account id is nil")
return nil, s3err.ErrInvalidRequest
}
_, ok := accountManager.IdNameMapping[*grantee.ID]
if !ok {
glog.Warningf("invalid canonical grantee! account id[%s] is not exists", *grantee.ID)
return nil, s3err.ErrInvalidRequest
}
result = append(result, grant)
case s3_constants.GrantTypeAmazonCustomerByEmail:
if grantee.EmailAddress == nil {
glog.Warning("invalid email grantee! email address is nil")
return nil, s3err.ErrInvalidRequest
}
accountId, ok := accountManager.EmailIdMapping[*grantee.EmailAddress]
if !ok {
glog.Warningf("invalid email grantee! email address[%s] is not exists", *grantee.EmailAddress)
return nil, s3err.ErrInvalidRequest
}
result = append(result, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: grant.Permission,
})
default:
return nil, s3err.ErrInvalidRequest
}
}
return result, s3err.ErrNone
}
// DetermineReqGrants generates the grant set (Grants) according to accountId and reqPermission.
func DetermineReqGrants(accountId, aclAction string) (grants []*s3.Grant) {
// group grantee (AllUsers)
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
})
// canonical grantee (accountId)
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
})
// group grantee (AuthenticateUsers)
if accountId != s3account.AccountAnonymous.Id {
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &aclAction,
})
grants = append(grants, &s3.Grant{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &s3_constants.PermissionFullControl,
})
}
return
}
func SetAcpOwnerHeader(r *http.Request, acpOwnerId string) {
r.Header.Set(s3_constants.ExtAmzOwnerKey, acpOwnerId)
}
func GetAcpOwner(entryExtended map[string][]byte, defaultOwner string) string {
ownerIdBytes, ok := entryExtended[s3_constants.ExtAmzOwnerKey]
if ok && len(ownerIdBytes) > 0 {
return string(ownerIdBytes)
}
return defaultOwner
}
func SetAcpGrantsHeader(r *http.Request, acpGrants []*s3.Grant) {
if len(acpGrants) > 0 {
a, err := json.Marshal(acpGrants)
if err == nil {
r.Header.Set(s3_constants.ExtAmzAclKey, string(a))
} else {
glog.Warning("Marshal acp grants err", err)
}
}
}
// GetAcpGrants return grants parsed from entry
func GetAcpGrants(entryExtended map[string][]byte) []*s3.Grant {
acpBytes, ok := entryExtended[s3_constants.ExtAmzAclKey]
if ok && len(acpBytes) > 0 {
var grants []*s3.Grant
err := json.Unmarshal(acpBytes, &grants)
if err == nil {
return grants
}
}
return nil
}
// AssembleEntryWithAcp fill entry with owner and grants
func AssembleEntryWithAcp(objectEntry *filer_pb.Entry, objectOwner string, grants []*s3.Grant) s3err.ErrorCode {
if objectEntry.Extended == nil {
objectEntry.Extended = make(map[string][]byte)
}
if len(objectOwner) > 0 {
objectEntry.Extended[s3_constants.ExtAmzOwnerKey] = []byte(objectOwner)
} else {
delete(objectEntry.Extended, s3_constants.ExtAmzOwnerKey)
}
if len(grants) > 0 {
grantsBytes, err := json.Marshal(grants)
if err != nil {
glog.Warning("assemble acp to entry:", err)
return s3err.ErrInvalidRequest
}
objectEntry.Extended[s3_constants.ExtAmzAclKey] = grantsBytes
} else {
delete(objectEntry.Extended, s3_constants.ExtAmzAclKey)
}
return s3err.ErrNone
}
// GrantEquals Compare whether two Grants are equal in meaning, not completely
// equal (compare Grantee.Type and the corresponding Value for equality, other
// fields of Grantee are ignored)
func GrantEquals(a, b *s3.Grant) bool {
// grant
if a == b {
return true
}
if a == nil || b == nil {
return false
}
// grant.Permission
if a.Permission != b.Permission {
if a.Permission == nil || b.Permission == nil {
return false
}
if *a.Permission != *b.Permission {
return false
}
}
// grant.Grantee
ag := a.Grantee
bg := b.Grantee
if ag != bg {
if ag == nil || bg == nil {
return false
}
// grantee.Type
if ag.Type != bg.Type {
if ag.Type == nil || bg.Type == nil {
return false
}
if *ag.Type != *bg.Type {
return false
}
}
// value corresponding to granteeType
if ag.Type != nil {
switch *ag.Type {
case s3_constants.GrantTypeGroup:
if ag.URI != bg.URI {
if ag.URI == nil || bg.URI == nil {
return false
}
if *ag.URI != *bg.URI {
return false
}
}
case s3_constants.GrantTypeCanonicalUser:
if ag.ID != bg.ID {
if ag.ID == nil || bg.ID == nil {
return false
}
if *ag.ID != *bg.ID {
return false
}
}
case s3_constants.GrantTypeAmazonCustomerByEmail:
if ag.EmailAddress != bg.EmailAddress {
if ag.EmailAddress == nil || bg.EmailAddress == nil {
return false
}
if *ag.EmailAddress != *bg.EmailAddress {
return false
}
}
}
}
}
return true
}
+706
View File
@@ -0,0 +1,706 @@
package s3acl
import (
"bytes"
"encoding/json"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"io"
"net/http"
"testing"
)
var (
accountManager = &s3account.AccountManager{
IdNameMapping: map[string]string{
s3account.AccountAdmin.Id: s3account.AccountAdmin.Name,
s3account.AccountAnonymous.Id: s3account.AccountAnonymous.Name,
"accountA": "accountA",
"accountB": "accountB",
},
EmailIdMapping: map[string]string{
s3account.AccountAdmin.EmailAddress: s3account.AccountAdmin.Id,
s3account.AccountAnonymous.EmailAddress: s3account.AccountAnonymous.Id,
"accountA@example.com": "accountA",
"accountBexample.com": "accountB",
},
}
)
func TestGetAccountId(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
//case1
//accountId: "admin"
req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAdmin.Id)
if GetAccountId(req) != s3account.AccountAdmin.Id {
t.Fatal("expect accountId: admin")
}
//case2
//accountId: "anoymous"
req.Header.Set(s3_constants.AmzAccountId, s3account.AccountAnonymous.Id)
if GetAccountId(req) != s3account.AccountAnonymous.Id {
t.Fatal("expect accountId: anonymous")
}
//case3
//accountId is nil => "anonymous"
req.Header.Del(s3_constants.AmzAccountId)
if GetAccountId(req) != s3account.AccountAnonymous.Id {
t.Fatal("expect accountId: anonymous")
}
}
func TestExtractAcl(t *testing.T) {
type Case struct {
id int
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
accountAdminId := "admin"
{
//case1 (good case)
//parse acp from request body
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
1,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountAdminId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse acp from header (cannedAcl)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
grants, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
2,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (bad case)
//parse acp from request body (content is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = io.NopCloser(bytes.NewReader([]byte("zdfsaf")))
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclPrivate)
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, accountAdminId, objectWriter)
testCases = append(testCases, &Case{
id: 3,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
//case4 (bad case)
//parse acp from header (cannedAcl is invalid)
req := &http.Request{
Header: make(map[string][]string),
}
req.Body = nil
req.Header.Set(s3_constants.AmzCannedAcl, "dfaksjfk")
objectWriter := "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, "", objectWriter)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
{
//case5 (bad case)
//parse acp from request body: owner is inconsistent
req.Body = io.NopCloser(bytes.NewReader([]byte(`
<AccessControlPolicy xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Owner>
<ID>admin</ID>
<DisplayName>admin</DisplayName>
</Owner>
<AccessControlList>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="CanonicalUser">
<ID>admin</ID>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
<Grant>
<Grantee xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="Group">
<URI>http://acs.amazonaws.com/groups/global/AllUsers</URI>
</Grantee>
<Permission>FULL_CONTROL</Permission>
</Grant>
</AccessControlList>
</AccessControlPolicy>
`)))
objectWriter = "accountA"
_, errCode := ExtractAcl(req, accountManager, s3_constants.OwnershipObjectWriter, accountAdminId, objectWriter, objectWriter)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrAccessDenied,
})
}
for _, tc := range testCases {
if tc.resultErrCode != tc.expectErrCode {
t.Fatalf("case[%d]: errorCode not expect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func TestParseAndValidateAclHeaders(t *testing.T) {
type Case struct {
id int
resultOwner, expectOwner string
resultErrCode, expectErrCode s3err.ErrorCode
resultGrants, expectGrants []*s3.Grant
}
testCases := make([]*Case, 0)
bucketOwner := "admin"
{
//case1 (good case)
//parse custom acl
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="anonymous", emailAddress="admin@example.com"`)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
1,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &s3account.AccountAnonymous.Id,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &s3account.AccountAdmin.Id,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case2 (good case)
//parse canned acl (ownership=ObjectWriter)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
2,
ownerId, objectWriter,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &objectWriter,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case3 (good case)
//parse canned acl (ownership=OwnershipBucketOwnerPreferred)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, s3_constants.CannedAclBucketOwnerFullControl)
ownerId, grants, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipBucketOwnerPreferred, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
3,
ownerId, bucketOwner,
errCode, s3err.ErrNone,
grants, []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &bucketOwner,
},
Permission: &s3_constants.PermissionFullControl,
},
},
})
}
{
//case4 (bad case)
//parse custom acl (grantee id not exists)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http://acs.amazonaws.com/groups/global/AllUsers", id="notExistsAccount", emailAddress="admin@example.com"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 4,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case5 (bad case)
//parse custom acl (invalid format)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzAclFullControl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
{
//case6 (bad case)
//parse canned acl (invalid value)
req := &http.Request{
Header: make(map[string][]string),
}
objectWriter := "accountA"
req.Header.Set(s3_constants.AmzCannedAcl, `uri="http:sfasf"`)
_, _, errCode := ParseAndValidateAclHeaders(req, accountManager, s3_constants.OwnershipObjectWriter, bucketOwner, objectWriter, false)
testCases = append(testCases, &Case{
id: 5,
resultErrCode: errCode, expectErrCode: s3err.ErrInvalidRequest,
})
}
for _, tc := range testCases {
if tc.expectErrCode != tc.resultErrCode {
t.Errorf("case[%d]: errCode unexpect", tc.id)
}
if tc.resultOwner != tc.expectOwner {
t.Errorf("case[%d]: ownerId unexpect", tc.id)
}
if !grantsEquals(tc.resultGrants, tc.expectGrants) {
t.Fatalf("case[%d]: grants not expect", tc.id)
}
}
}
func grantsEquals(a, b []*s3.Grant) bool {
if len(a) != len(b) {
return false
}
for i, grant := range a {
if !GrantEquals(grant, b[i]) {
return false
}
}
return true
}
func TestDetermineReqGrants(t *testing.T) {
{
//case1: request account is anonymous
accountId := s3account.AccountAnonymous.Id
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
{
//case2: request account is not anonymous (Iam authed)
accountId := "accountX"
reqPermission := s3_constants.PermissionRead
resultGrants := DetermineReqGrants(accountId, reqPermission)
expectGrants := []*s3.Grant{
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeCanonicalUser,
ID: &accountId,
},
Permission: &s3_constants.PermissionFullControl,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &reqPermission,
},
{
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAuthenticatedUsers,
},
Permission: &s3_constants.PermissionFullControl,
},
}
if !grantsEquals(resultGrants, expectGrants) {
t.Fatalf("grants not expect")
}
}
}
func TestAssembleEntryWithAcp(t *testing.T) {
defaultOwner := "admin"
//case1
//assemble with non-empty grants
expectOwner := "accountS"
expectGrants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
entry := &filer_pb.Entry{}
AssembleEntryWithAcp(entry, expectOwner, expectGrants)
resultOwner := GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != expectOwner {
t.Fatalf("owner not expect")
}
resultGrants := GetAcpGrants(entry.Extended)
if !grantsEquals(resultGrants, expectGrants) {
t.Fatal("grants not expect")
}
//case2
//assemble with empty grants (override)
AssembleEntryWithAcp(entry, "", nil)
resultOwner = GetAcpOwner(entry.Extended, defaultOwner)
if resultOwner != defaultOwner {
t.Fatalf("owner not expect")
}
resultGrants = GetAcpGrants(entry.Extended)
if len(resultGrants) != 0 {
t.Fatal("grants not expect")
}
}
func TestGrantEquals(t *testing.T) {
testCases := map[bool]bool{
GrantEquals(nil, nil): true,
GrantEquals(&s3.Grant{}, nil): false,
GrantEquals(&s3.Grant{}, &s3.Grant{}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{},
}): false,
//type not present, compare other fields of grant is meaningless
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: &s3account.AccountAdmin.Id,
EmailAddress: &s3account.AccountAdmin.EmailAddress,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
ID: &s3account.AccountAdmin.Id,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionWrite,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
},
}): true,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
},
}): false,
GrantEquals(&s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
URI: &s3_constants.GranteeGroupAllUsers,
},
}, &s3.Grant{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
URI: &s3_constants.GranteeGroupAllUsers,
},
}): true,
}
for tc, expect := range testCases {
if tc != expect {
t.Fatal("TestGrantEquals not expect!")
}
}
}
func TestSetAcpOwnerHeader(t *testing.T) {
ownerId := "accountZ"
req := &http.Request{
Header: make(map[string][]string),
}
SetAcpOwnerHeader(req, ownerId)
if req.Header.Get(s3_constants.ExtAmzOwnerKey) != ownerId {
t.Fatalf("owner unexpect")
}
}
func TestSetAcpGrantsHeader(t *testing.T) {
req := &http.Request{
Header: make(map[string][]string),
}
grants := []*s3.Grant{
{
Permission: &s3_constants.PermissionRead,
Grantee: &s3.Grantee{
Type: &s3_constants.GrantTypeGroup,
ID: &s3account.AccountAdmin.Id,
URI: &s3_constants.GranteeGroupAllUsers,
},
},
}
SetAcpGrantsHeader(req, grants)
grantsJson, _ := json.Marshal(grants)
if req.Header.Get(s3_constants.ExtAmzAclKey) != string(grantsJson) {
t.Fatalf("owner unexpect")
}
}
+29
View File
@@ -0,0 +1,29 @@
package s3api
import (
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"net/http"
)
func getAccountId(r *http.Request) string {
id := r.Header.Get(s3_constants.AmzAccountId)
if len(id) == 0 {
return s3account.AccountAnonymous.Id
} else {
return id
}
}
func (s3a *S3ApiServer) checkAccessByOwnership(r *http.Request, bucket string) s3err.ErrorCode {
metadata, errCode := s3a.bucketRegistry.GetBucketMetadata(bucket)
if errCode != s3err.ErrNone {
return errCode
}
accountId := getAccountId(r)
if accountId == s3account.AccountAdmin.Id || accountId == *metadata.Owner.ID {
return s3err.ErrNone
}
return s3err.ErrAccessDenied
}
+158
View File
@@ -5,6 +5,8 @@ import (
"encoding/xml"
"errors"
"fmt"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"net/http"
"time"
@@ -343,3 +345,159 @@ func (s3a *S3ApiServer) GetBucketLocationHandler(w http.ResponseWriter, r *http.
func (s3a *S3ApiServer) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
writeSuccessResponseXML(w, r, RequestPaymentConfiguration{Payer: "BucketOwner"})
}
// PutBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html
func (s3a *S3ApiServer) PutBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
if r.Body == nil || r.Body == http.NoBody {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
var v s3.OwnershipControls
defer util.CloseRequest(r)
err := xmlutil.UnmarshalXML(&v, xml.NewDecoder(r.Body), "")
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
if len(v.Rules) != 1 {
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
printOwnership := true
ownership := *v.Rules[0].ObjectOwnership
switch ownership {
case s3_constants.OwnershipObjectWriter:
case s3_constants.OwnershipBucketOwnerPreferred:
case s3_constants.OwnershipBucketOwnerEnforced:
printOwnership = false
default:
s3err.WriteErrorResponse(w, r, s3err.ErrInvalidRequest)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
oldOwnership, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok || string(oldOwnership) != ownership {
if bucketEntry.Extended == nil {
bucketEntry.Extended = make(map[string][]byte)
}
bucketEntry.Extended[s3_constants.ExtOwnershipKey] = []byte(ownership)
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
}
if printOwnership {
result := &s3.PutBucketOwnershipControlsInput{
OwnershipControls: &v,
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
} else {
writeSuccessResponseEmpty(w, r)
}
}
// GetBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html
func (s3a *S3ApiServer) GetBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("GetBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
v, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok {
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
return
}
ownership := string(v)
result := &s3.PutBucketOwnershipControlsInput{
OwnershipControls: &s3.OwnershipControls{
Rules: []*s3.OwnershipControlsRule{
{
ObjectOwnership: &ownership,
},
},
},
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, result)
}
// DeleteBucketOwnershipControls https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html
func (s3a *S3ApiServer) DeleteBucketOwnershipControls(w http.ResponseWriter, r *http.Request) {
bucket, _ := s3_constants.GetBucketAndObject(r)
glog.V(3).Infof("PutBucketOwnershipControls %s", bucket)
errCode := s3a.checkAccessByOwnership(r, bucket)
if errCode != s3err.ErrNone {
s3err.WriteErrorResponse(w, r, errCode)
return
}
bucketEntry, err := s3a.getEntry(s3a.option.BucketsPath, bucket)
if err != nil {
if err == filer_pb.ErrNotFound {
s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
return
}
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
_, ok := bucketEntry.Extended[s3_constants.ExtOwnershipKey]
if !ok {
s3err.WriteErrorResponse(w, r, s3err.OwnershipControlsNotFoundError)
return
}
delete(bucketEntry.Extended, s3_constants.ExtOwnershipKey)
err = s3a.updateEntry(s3a.option.BucketsPath, bucketEntry)
if err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
return
}
emptyOwnershipControls := &s3.OwnershipControls{
Rules: []*s3.OwnershipControlsRule{},
}
s3err.WriteAwsXMLResponse(w, r, http.StatusOK, emptyOwnershipControls)
}
+14 -1
View File
@@ -5,6 +5,7 @@ import (
"fmt"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3account"
"net"
"net/http"
"strings"
@@ -40,6 +41,8 @@ type S3ApiServer struct {
randomClientId int32
filerGuard *security.Guard
client *http.Client
accountManager *s3account.AccountManager
bucketRegistry *BucketRegistry
}
func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer *S3ApiServer, err error) {
@@ -59,6 +62,8 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
filerGuard: security.NewGuard([]string{}, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec),
cb: NewCircuitBreaker(option),
}
s3ApiServer.accountManager = s3account.NewAccountManager(s3ApiServer)
s3ApiServer.bucketRegistry = NewBucketRegistry(s3ApiServer)
if option.LocalFilerSocket == "" {
s3ApiServer.client = &http.Client{Transport: &http.Transport{
MaxIdleConns: 1024,
@@ -76,7 +81,7 @@ func NewS3ApiServer(router *mux.Router, option *S3ApiServerOption) (s3ApiServer
s3ApiServer.registerRouter(router)
go s3ApiServer.subscribeMetaEvents("s3", filer.DirectoryEtcRoot, time.Now().UnixNano())
go s3ApiServer.subscribeMetaEvents("s3", time.Now().UnixNano(), filer.DirectoryEtcRoot, []string{option.BucketsPath})
return s3ApiServer, nil
}
@@ -212,6 +217,14 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) {
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV2Handler, ACTION_LIST)), "LIST")).Queries("list-type", "2")
// buckets with query
// PutBucketOwnershipControls
bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.PutBucketOwnershipControls, ACTION_ADMIN), "PUT")).Queries("ownershipControls", "")
//GetBucketOwnershipControls
bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.GetBucketOwnershipControls, ACTION_READ), "GET")).Queries("ownershipControls", "")
//DeleteBucketOwnershipControls
bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.DeleteBucketOwnershipControls, ACTION_ADMIN), "DELETE")).Queries("ownershipControls", "")
// raw buckets
+11
View File
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/xml"
"fmt"
"github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"net/http"
@@ -19,6 +20,16 @@ const (
MimeXML mimeType = "application/xml"
)
func WriteAwsXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, result interface{}) {
var bytesBuffer bytes.Buffer
err := xmlutil.BuildXML(result, xml.NewEncoder(&bytesBuffer))
if err != nil {
WriteErrorResponse(w, r, ErrInternalError)
return
}
WriteResponse(w, r, statusCode, bytesBuffer.Bytes(), MimeXML)
}
func WriteXMLResponse(w http.ResponseWriter, r *http.Request, statusCode int, response interface{}) {
WriteResponse(w, r, statusCode, EncodeXMLResponse(response), MimeXML)
}
+8
View File
@@ -107,6 +107,8 @@ const (
ErrTooManyRequest
ErrRequestBytesExceed
OwnershipControlsNotFoundError
)
// error code to APIError structure, these fields carry respective
@@ -414,6 +416,12 @@ var errorCodeResponse = map[ErrorCode]APIError{
Description: "Simultaneous request bytes exceed limitations",
HTTPStatusCode: http.StatusTooManyRequests,
},
OwnershipControlsNotFoundError: {
Code: "OwnershipControlsNotFoundError",
Description: "The bucket ownership controls were not found",
HTTPStatusCode: http.StatusNotFound,
},
}
// GetAPIError provides API Error for input API error code.
+6
View File
@@ -33,6 +33,12 @@ func track(f http.HandlerFunc, action string) http.HandlerFunc {
recorder := NewStatusResponseWriter(w)
start := time.Now()
f(recorder, r)
if recorder.Status == http.StatusForbidden {
if m, _ := stats_collect.S3RequestCounter.GetMetricWithLabelValues(
action, strconv.Itoa(http.StatusOK), bucket); m == nil {
bucket = ""
}
}
stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
}
+24 -11
View File
@@ -16,6 +16,15 @@ import (
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
start := time.Now()
if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Expose-Headers", "*")
w.Header().Set("Access-Control-Allow-Headers", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true")
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
}
if r.Method == "OPTIONS" {
OptionsHandler(w, r, false)
return
@@ -45,10 +54,7 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
}()
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true")
}
switch r.Method {
case "GET":
fs.GetOrHeadHandler(w, r)
@@ -92,6 +98,13 @@ func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Request) {
start := time.Now()
if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Headers", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true")
}
stats.FilerRequestCounter.WithLabelValues(r.Method).Inc()
defer func() {
stats.FilerRequestHistogram.WithLabelValues(r.Method).Observe(time.Since(start).Seconds())
@@ -108,10 +121,7 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
}
w.Header().Set("Server", "SeaweedFS Filer "+util.VERSION)
if r.Header.Get("Origin") != "" {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true")
}
switch r.Method {
case "GET":
fs.GetOrHeadHandler(w, r)
@@ -122,11 +132,14 @@ func (fs *FilerServer) readonlyFilerHandler(w http.ResponseWriter, r *http.Reque
func OptionsHandler(w http.ResponseWriter, r *http.Request, isReadOnly bool) {
if isReadOnly {
w.Header().Add("Access-Control-Allow-Methods", "GET, OPTIONS")
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
} else {
w.Header().Add("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Methods", "PUT, POST, GET, DELETE, OPTIONS")
w.Header().Set("Access-Control-Expose-Headers", "*")
}
w.Header().Add("Access-Control-Allow-Headers", "*")
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Headers", "*")
w.Header().Set("Access-Control-Allow-Credentials", "true")
}
// maybeCheckJwtAuthorization returns true if access should be granted, false if it should be denied
+7 -2
View File
@@ -107,9 +107,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
return
}
query := r.URL.Query()
if entry.IsDirectory() {
if fs.option.DisableDirListing {
w.WriteHeader(http.StatusMethodNotAllowed)
w.WriteHeader(http.StatusForbidden)
return
}
if query.Get("metadata") == "true" {
writeJsonQuiet(w, r, http.StatusOK, entry)
return
}
if entry.Attr.Mime == "" {
@@ -125,7 +131,6 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
return
}
query := r.URL.Query()
if query.Get("metadata") == "true" {
if query.Get("resolveManifest") == "true" {
if entry.Chunks, _, err = filer.ResolveChunkManifest(
@@ -3,6 +3,7 @@ package weed_server
import (
"context"
"fmt"
//"github.com/seaweedfs/seaweedfs/weed/s3api"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"io"
"net/http"
@@ -374,6 +375,18 @@ func SaveAmzMetaData(r *http.Request, existing map[string][]byte, isReplace bool
}
}
//acp-owner
acpOwner := r.Header.Get(s3_constants.ExtAmzOwnerKey)
if len(acpOwner) > 0 {
metadata[s3_constants.ExtAmzOwnerKey] = []byte(acpOwner)
}
//acp-grants
acpGrants := r.Header.Get(s3_constants.ExtAmzAclKey)
if len(acpOwner) > 0 {
metadata[s3_constants.ExtAmzAclKey] = []byte(acpGrants)
}
return
}
+2
View File
@@ -307,6 +307,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
case <-ticker.C:
if !ms.Topo.IsLeader() {
stats.MasterRaftIsleader.Set(0)
stats.MasterAdminLock.Reset()
stats.MasterReplicaPlacementMismatch.Reset()
return ms.informNewLeader(stream)
} else {
stats.MasterRaftIsleader.Set(1)
+3
View File
@@ -3,6 +3,7 @@ package weed_server
import (
"context"
"fmt"
"github.com/seaweedfs/seaweedfs/weed/stats"
"math/rand"
"sync"
"time"
@@ -112,11 +113,13 @@ func (locks *AdminLocks) generateToken(lockName string, clientName string) (ts t
lastClient: clientName,
}
locks.locks[lockName] = lock
stats.MasterAdminLock.WithLabelValues(clientName).Set(1)
return lock.accessLockTime, lock.accessSecret
}
func (locks *AdminLocks) deleteLock(lockName string) {
locks.Lock()
stats.MasterAdminLock.WithLabelValues(locks.locks[lockName].lastClient).Set(0)
defer locks.Unlock()
delete(locks.locks, lockName)
}
+2
View File
@@ -20,6 +20,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
}
servers := ms.Topo.HashicorpRaft.GetConfiguration().Configuration().Servers
_, leaderId := ms.Topo.HashicorpRaft.LeaderWithID()
ms.Topo.RaftServerAccessLock.RUnlock()
for _, server := range servers {
@@ -27,6 +28,7 @@ func (ms *MasterServer) RaftListClusterServers(ctx context.Context, req *master_
Id: string(server.ID),
Address: string(server.Address),
Suffrage: server.Suffrage.String(),
IsLeader: server.ID == leaderId,
})
}
return resp, nil
+26 -11
View File
@@ -174,39 +174,45 @@ func (ms *MasterServer) SetRaftServer(raftServer *RaftServer) {
glog.V(0).Infof("leader change event: %+v => %+v", e.PrevValue(), e.Value())
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", e.Value())).Inc()
if ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "becomes leader.")
glog.V(0).Infof("[%s] %s becomes leader.", ms.Topo.RaftServer.Name(), ms.Topo.RaftServer.Leader())
}
})
raftServerName = ms.Topo.RaftServer.Name()
raftServerName = fmt.Sprintf("[%s]", ms.Topo.RaftServer.Name())
} else if raftServer.RaftHashicorp != nil {
ms.Topo.HashicorpRaft = raftServer.RaftHashicorp
leaderCh := raftServer.RaftHashicorp.LeaderCh()
prevLeader := ms.Topo.HashicorpRaft.Leader()
prevLeader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
raftServerName = ms.Topo.HashicorpRaft.String()
go func() {
for {
select {
case isLeader := <-leaderCh:
leader := ms.Topo.HashicorpRaft.Leader()
ms.Topo.RaftServerAccessLock.RLock()
leader, _ := ms.Topo.HashicorpRaft.LeaderWithID()
ms.Topo.RaftServerAccessLock.RUnlock()
glog.V(0).Infof("is leader %+v change event: %+v => %+v", isLeader, prevLeader, leader)
stats.MasterLeaderChangeCounter.WithLabelValues(fmt.Sprintf("%+v", leader)).Inc()
prevLeader = leader
}
}
}()
raftServerName = ms.Topo.HashicorpRaft.String()
}
ms.Topo.RaftServerAccessLock.Unlock()
if ms.Topo.IsLeader() {
glog.V(0).Infoln("[", raftServerName, "]", "I am the leader!")
glog.V(0).Infof("%s I am the leader!", raftServerName)
} else {
var raftServerLeader string
ms.Topo.RaftServerAccessLock.RLock()
if ms.Topo.RaftServer != nil && ms.Topo.RaftServer.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.RaftServer.Name(), "]", ms.Topo.RaftServer.Leader(), "is the leader.")
} else if ms.Topo.HashicorpRaft != nil && ms.Topo.HashicorpRaft.Leader() != "" {
glog.V(0).Infoln("[", ms.Topo.HashicorpRaft.String(), "]", ms.Topo.HashicorpRaft.Leader(), "is the leader.")
if ms.Topo.RaftServer != nil {
raftServerLeader = ms.Topo.RaftServer.Leader()
} else if ms.Topo.HashicorpRaft != nil {
raftServerName = ms.Topo.HashicorpRaft.String()
raftServerLeaderAddr, _ := ms.Topo.HashicorpRaft.LeaderWithID()
raftServerLeader = string(raftServerLeaderAddr)
}
ms.Topo.RaftServerAccessLock.RUnlock()
glog.V(0).Infof("%s %s - is the leader.", raftServerName, raftServerLeader)
}
}
@@ -388,8 +394,17 @@ func (ms *MasterServer) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, startF
} else {
glog.V(0).Infof("master %s successfully responded to ping", peerName)
}
return nil
})
}
}
func (ms *MasterServer) Shutdown() {
if ms.Topo == nil || ms.Topo.HashicorpRaft == nil {
return
}
if ms.Topo.HashicorpRaft.State() == hashicorpRaft.Leader {
ms.Topo.HashicorpRaft.LeadershipTransfer()
}
ms.Topo.HashicorpRaft.Shutdown()
}
+16
View File
@@ -6,10 +6,13 @@ package weed_server
import (
"fmt"
transport "github.com/Jille/raft-grpc-transport"
"github.com/armon/go-metrics"
"github.com/armon/go-metrics/prometheus"
"github.com/hashicorp/raft"
boltdb "github.com/hashicorp/raft-boltdb/v2"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"google.golang.org/grpc"
"math/rand"
"os"
@@ -182,5 +185,18 @@ func NewHashicorpRaftServer(option *RaftServerOption) (*RaftServer, error) {
}()
}
// Configure a prometheus sink as the raft metrics sink
if sink, err := prometheus.NewPrometheusSinkFrom(prometheus.PrometheusOpts{
Registerer: stats.Gather,
}); err != nil {
return nil, fmt.Errorf("NewPrometheusSink: %v", err)
} else {
metricsConf := metrics.DefaultConfig(stats.Namespace)
metricsConf.EnableRuntimeMetrics = false
if _, err = metrics.NewGlobal(metricsConf, sink); err != nil {
return nil, fmt.Errorf("metrics.NewGlobal: %v", err)
}
}
return s, nil
}
Loaded 100 of 140 files, more files were not shown because too many files have changed in this diff. Show more